Esempio n. 1
0
func (c *ArchiveCommand) processUntracked() {
	if !c.Untracked {
		return
	}

	var output bytes.Buffer
	cmd := exec.Command("git", "ls-files", "-o")
	cmd.Env = os.Environ()
	cmd.Stdout = &output
	cmd.Stderr = os.Stderr
	logrus.Debugln("Executing command:", strings.Join(cmd.Args, " "))
	err := cmd.Run()
	if err == nil {
		reader := bufio.NewReader(&output)
		for {
			line, _, err := reader.ReadLine()
			if err == io.EOF {
				break
			} else if err != nil {
				logrus.Warningln(err)
				break
			}
			c.process(string(line))
		}
	} else {
		logrus.Warningln(err)
	}
}
func createZipEntry(archive *zip.Writer, fileName string) error {
	fi, err := os.Lstat(fileName)
	if err != nil {
		logrus.Warningln("File ignored:", err)
		return nil
	}

	fh, err := zip.FileInfoHeader(fi)
	if err != nil {
		return err
	}
	fh.Name = fileName
	fh.Extra = createZipExtra(fi)

	switch fi.Mode() & os.ModeType {
	case os.ModeDir:
		return createZipDirectoryEntry(archive, fh)

	case os.ModeSymlink:
		return createZipSymlinkEntry(archive, fh)

	case os.ModeNamedPipe, os.ModeSocket, os.ModeDevice:
		// Ignore the files that of these types
		logrus.Warningln("File ignored:", fileName)
		return nil

	default:
		return createZipFileEntry(archive, fh)
	}
}
Esempio n. 3
0
func waitForInterrupts(finished *bool, abortSignal chan os.Signal, doneSignal chan int) {
	signals := make(chan os.Signal)
	signal.Notify(signals, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)

	interrupt := <-signals
	if finished != nil {
		*finished = true
	}

	// request stop, but wait for force exit
	for interrupt == syscall.SIGQUIT {
		log.Warningln("Requested quit, waiting for builds to finish")
		interrupt = <-signals
	}

	log.Warningln("Requested exit:", interrupt)

	go func() {
		for {
			abortSignal <- interrupt
		}
	}()

	select {
	case newSignal := <-signals:
		log.Fatalln("forced exit:", newSignal)
	case <-time.After(common.ShutdownTimeout * time.Second):
		log.Fatalln("shutdown timedout")
	case <-doneSignal:
	}
}
Esempio n. 4
0
func (n *GitLabClient) UpdateBuild(config RunnerConfig, id int, state BuildState, trace string) UpdateState {
	request := UpdateBuildRequest{
		Info:  n.getRunnerVersion(config),
		Token: config.Token,
		State: state,
		Trace: trace,
	}

	result, statusText := n.do(config.RunnerCredentials, "PUT", fmt.Sprintf("builds/%d.json", id), 200, &request, nil)
	switch result {
	case 200:
		logrus.Println(config.ShortDescription(), id, "Submitting build to coordinator...", "ok")
		return UpdateSucceeded
	case 404:
		logrus.Warningln(config.ShortDescription(), id, "Submitting build to coordinator...", "aborted")
		return UpdateAbort
	case 403:
		logrus.Errorln(config.ShortDescription(), id, "Submitting build to coordinator...", "forbidden")
		return UpdateAbort
	case clientError:
		logrus.Errorln(config.ShortDescription(), id, "Submitting build to coordinator...", "error", statusText)
		return UpdateAbort
	default:
		logrus.Warningln(config.ShortDescription(), id, "Submitting build to coordinator...", "failed", statusText)
		return UpdateFailed
	}
}
Esempio n. 5
0
/*
Install a package with apt-get
*/
func (pkg *Package) installAptPackage(name string) error {
	out, err := exec.Command("apt-get", "update").CombinedOutput()
	if err != nil {
		log.Warningln(string(out))
		return err
	}
	out, err = exec.Command("apt-get", "install", "-y", name).CombinedOutput()
	if err != nil {
		log.Warningln(string(out))
		return err
	}
	log.Println("Installed Apt package: ", string(out))
	return nil
}
func (c *fileArchiver) processPaths() {
	for _, path := range c.Paths {
		matches, err := filepath.Glob(path)
		if err != nil {
			logrus.Warningf("%s: %v", path, err)
			continue
		}

		found := 0

		for _, match := range matches {
			err := filepath.Walk(match, func(path string, info os.FileInfo, err error) error {
				if c.process(path) {
					found++
				}
				return nil
			})
			if err != nil {
				logrus.Warningln("Walking", match, err)
			}
		}

		if found == 0 {
			logrus.Warningf("%s: no matching files", path)
		} else {
			logrus.Infof("%s: found %d matching files", path, found)
		}
	}
}
Esempio n. 7
0
func dispatchCommand(c *cli.Context, client *etcd.Client, cmd *command.Command) {
	targets := c.StringSlice("target")
	if targets == nil || len(targets) == 0 {
		log.Warningln("no target set! try to send command to all registered host.")
		targets = fetchHosts(c, client)
	}
	if targets == nil {
		log.Fatalln("no target to send command.")
	} else {
		log.Infoln("send command to: ", targets)
	}
	for _, target := range targets {
		key := fmt.Sprintf("/beacon/commands/single/%s/%s/",
			target, cmd.Id)
		if c.GlobalString("prefix") != "" {
			key = fmt.Sprintf("/%s%s", strings.Trim(c.GlobalString("prefix"), "/"), key)
		}

		if _, err := client.Set(key, cmd.Marshal(), 0); err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Fatalln("send command failed.")
		}
	}
}
func (c *UnregisterCommand) Execute(context *cli.Context) {
	if !common.DeleteRunner(c.URL, c.Token) {
		log.Fatalln("Failed to delete runner")
	}

	err := c.loadConfig()
	if err != nil {
		log.Warningln(err)
		return
	}

	runners := []*common.RunnerConfig{}
	for _, otherRunner := range c.config.Runners {
		if otherRunner.RunnerCredentials == c.RunnerCredentials {
			continue
		}
		runners = append(runners, otherRunner)
	}

	// check if anything changed
	if len(c.config.Runners) == len(runners) {
		return
	}

	c.config.Runners = runners

	// save config file
	err = c.saveConfig()
	if err != nil {
		log.Fatalln("Failed to update", c.ConfigFile, err)
	}
	log.Println("Updated", c.ConfigFile)
}
Esempio n. 9
0
func (n *GitLabClient) GetBuild(config RunnerConfig) (*GetBuildResponse, bool) {
	request := GetBuildRequest{
		Info:  n.getRunnerVersion(config),
		Token: config.Token,
	}

	var response GetBuildResponse
	result, statusText := n.do(config.RunnerCredentials, "POST", "builds/register.json", 201, &request, &response)

	switch result {
	case 201:
		logrus.Println(config.ShortDescription(), "Checking for builds...", "received")
		return &response, true
	case 403:
		logrus.Errorln(config.ShortDescription(), "Checking for builds...", "forbidden")
		return nil, false
	case 404:
		logrus.Debugln(config.ShortDescription(), "Checking for builds...", "nothing")
		return nil, true
	case clientError:
		logrus.Errorln(config.ShortDescription(), "Checking for builds...", "error:", statusText)
		return nil, false
	default:
		logrus.Warningln(config.ShortDescription(), "Checking for builds...", "failed:", statusText)
		return nil, true
	}
}
Esempio n. 10
0
func (c *ExecCommand) createBuild(repoURL string, abortSignal chan os.Signal) (build *common.Build, err error) {
	// Check if we have uncommitted changes
	_, err = c.runCommand("git", "diff", "--quiet", "HEAD")
	if err != nil {
		logrus.Warningln("You most probably have uncommitted changes.")
		logrus.Warningln("These changes will not be tested.")
	}

	// Parse Git settings
	sha, err := c.runCommand("git", "rev-parse", "HEAD")
	if err != nil {
		return
	}

	beforeSha, err := c.runCommand("git", "rev-parse", "HEAD~1")
	if err != nil {
		beforeSha = "0000000000000000000000000000000000000000"
	}

	refName, err := c.runCommand("git", "rev-parse", "--abbrev-ref", "HEAD")
	if err != nil {
		return
	}

	build = &common.Build{
		GetBuildResponse: common.GetBuildResponse{
			ID:            1,
			ProjectID:     1,
			RepoURL:       repoURL,
			Commands:      "",
			Sha:           strings.TrimSpace(sha),
			RefName:       strings.TrimSpace(refName),
			BeforeSha:     strings.TrimSpace(beforeSha),
			AllowGitFetch: false,
			Timeout:       c.getTimeout(),
			Token:         "",
			Name:          "",
			Stage:         "",
			Tag:           false,
		},
		Runner: &common.RunnerConfig{
			RunnerSettings: c.RunnerSettings,
		},
		SystemInterrupt: abortSignal,
	}
	return
}
Esempio n. 11
0
func (mr *RunCommand) processRunner(id int, runner *common.RunnerConfig, runners chan *common.RunnerConfig) (err error) {
	provider := common.GetExecutor(runner.Executor)
	if provider == nil {
		return
	}

	context, err := provider.Acquire(runner)
	if err != nil {
		log.Warningln("Failed to update executor", runner.Executor, "for", runner.ShortDescription(), err)
		return
	}
	defer provider.Release(runner, context)

	// Acquire build slot
	if !mr.buildsHelper.acquire(runner) {
		return
	}
	defer mr.buildsHelper.release(runner)

	// Receive a new build
	buildData, healthy := mr.network.GetBuild(*runner)
	mr.makeHealthy(runner.UniqueID(), healthy)
	if buildData == nil {
		return
	}

	// Make sure to always close output
	buildCredentials := &common.BuildCredentials{
		ID:    buildData.ID,
		Token: buildData.Token,
	}
	trace := mr.network.ProcessBuild(*runner, buildCredentials)
	defer trace.Fail(err)

	// Create a new build
	build := &common.Build{
		GetBuildResponse: *buildData,
		Runner:           runner,
		ExecutorData:     context,
		SystemInterrupt:  mr.abortBuilds,
	}

	// Add build to list of builds to assign numbers
	mr.buildsHelper.addBuild(build)
	defer mr.buildsHelper.removeBuild(build)

	// Process the same runner by different worker again
	// to speed up taking the builds
	select {
	case runners <- runner:
		mr.log().WithField("runner", runner.ShortDescription()).Debugln("Requeued the runner")

	default:
		mr.log().WithField("runner", runner.ShortDescription()).Debugln("Failed to requeue the runner: ")
	}

	// Process a build
	return build.Run(mr.config, trace)
}
Esempio n. 12
0
func (c *ArchiveCommand) processPaths() {
	for _, path := range c.Paths {
		matches, err := filepath.Glob(path)
		if err != nil {
			logrus.Warningln(err)
			continue
		}

		for _, match := range matches {
			err := filepath.Walk(match, func(path string, info os.FileInfo, err error) error {
				return c.process(path)
			})
			if err != nil {
				logrus.Warningln("Walking", match, err)
			}
		}
	}
}
func (e *AbstractExecutor) Warningln(args ...interface{}) {
	// write to log file
	if e.Build != nil {
		e.Build.WriteString(helpers.ANSI_BOLD_YELLOW + "WARNING: " + fmt.Sprintln(args...) + helpers.ANSI_RESET)
	}

	args = append([]interface{}{e.Config.ShortDescription(), e.Build.ID}, args...)
	log.Warningln(args...)
}
Esempio n. 14
0
func getCacheStorageClient(cache *common.CacheConfig) (scl *minio.Client, err error) {
	scl, err = minio.New(cache.ServerAddress, cache.AccessKey, cache.SecretKey, cache.Insecure)
	if err != nil {
		logrus.Warningln(err)
		return
	}

	scl.SetCustomTransport(&bucketLocationTripper{cache.BucketLocation})
	return
}
func (m *testNetwork) UploadRawArtifacts(config common.BuildCredentials, reader io.Reader, baseName string, expireIn string) common.UploadState {
	m.uploadCalled++

	if m.uploadState == common.UploadSucceeded {
		var buffer bytes.Buffer
		io.Copy(&buffer, reader)
		archive, err := zip.NewReader(bytes.NewReader(buffer.Bytes()), int64(buffer.Len()))
		if err != nil {
			logrus.Warningln(err)
			return common.UploadForbidden
		}

		if len(archive.File) != 1 || archive.File[0].Name != artifactsTestArchivedFile {
			logrus.Warningln("Invalid archive:", len(archive.File))
			return common.UploadForbidden
		}
	}
	return m.uploadState
}
func (r *retryHelper) doRetry(handler func() (bool, error)) (err error) {
	retry, err := handler()
	for i := 0; retry && i < r.Retry; i++ {
		// wait one second to retry
		logrus.Warningln("Retrying...")
		time.Sleep(r.RetryTime)
		retry, err = handler()
	}
	return
}
Esempio n. 17
0
// place a check on the queue
func (sm *Manager) enqueueCheck(check stalker.Check) {
	cb, err := json.Marshal(check)
	stalker.OnlyLogIf(err)
	rconn := sm.rpool.Get()
	defer rconn.Close()
	res, err := rconn.Do("RPUSH", "worker1", cb)
	log.Debugln("Checks now on queue:", res)
	if err != nil {
		log.Warningln("error pushing check on queue:", err)
	}
}
func (c *fileArchiver) isFileChanged(fileName string) bool {
	ai, err := os.Stat(fileName)
	if ai != nil {
		if !c.isChanged(ai.ModTime()) {
			return false
		}
	} else if !os.IsNotExist(err) {
		logrus.Warningln(err)
	}
	return true
}
Esempio n. 19
0
// Check if pause file exists and sleep until its removed if it does
func (sm *Manager) pauseIfAsked() {
	if stalker.FExists(sm.pauseFile) {
		log.Warningln("Pausing")
		for {
			time.Sleep(1 * time.Second)
			if !stalker.FExists(sm.pauseFile) {
				return
			}
		}
	}
}
Esempio n. 20
0
func getS3UploadURL(build *common.Build, key string) (url *url.URL) {
	cache := build.Runner.Cache
	objectName := getCacheObjectName(build, cache, key)
	if objectName == "" {
		return
	}

	scl, err := getCacheStorageClient(cache)
	if err != nil {
		logrus.Warningln(err)
		return
	}

	url, err = scl.PresignedPutObject(cache.BucketName, objectName, time.Second*time.Duration(build.Timeout))
	if err != nil {
		logrus.Warningln(err)
		return
	}
	return
}
func New(i service.Interface, c *service.Config) (service.Service, error) {
	s, err := service.New(i, c)
	if err == service.ErrNoServiceSystemDetected {
		log.Warningln("No service system detected. Some features may not work!")

		return &SimpleService{
			i: i,
			c: c,
		}, nil
	}
	return s, err
}
Esempio n. 22
0
func (c *ArchiveCommand) add(path string, info os.FileInfo) (err error) {
	if info == nil {
		info, err = os.Lstat(path)
	}
	if err == nil {
		c.files[path] = info
	} else if os.IsNotExist(err) {
		logrus.Warningln("File", path, "doesn't exist")
		err = nil
	}
	return
}
Esempio n. 23
0
//Push image to Clair for analysis
func Push(image Image) error {
	layerCount := len(image.FsLayers)

	parentID := ""

	if layerCount == 0 {
		logrus.Warningln("there is no layer to push")
	}
	localIP, err := config.LocalServerIP()
	if err != nil {
		return err
	}
	hURL := fmt.Sprintf("http://%v/v2", localIP)
	if IsLocal {
		hURL = strings.Replace(hURL, "/v2", "/local", -1)
		logrus.Infof("using %v as local url", hURL)
	}

	for index, layer := range image.FsLayers {
		lUID := xstrings.Substr(layer.BlobSum, 0, 12)
		logrus.Infof("Pushing Layer %d/%d [%v]", index+1, layerCount, lUID)

		database.InsertRegistryMapping(layer.BlobSum, image.Registry)
		payload := v1.LayerEnvelope{Layer: &v1.Layer{
			Name:       layer.BlobSum,
			Path:       image.BlobsURI(layer.BlobSum),
			ParentName: parentID,
			Format:     "Docker",
		}}

		//FIXME Update to TLS
		if IsLocal {
			payload.Layer.Name = layer.History
			payload.Layer.Path += "/layer.tar"
		}
		payload.Layer.Path = strings.Replace(payload.Layer.Path, image.Registry, hURL, 1)
		if err := clair.Push(payload); err != nil {
			logrus.Infof("adding layer %d/%d [%v]: %v", index+1, layerCount, lUID, err)
			if err != clair.OSNotSupported {
				return err
			}
			parentID = ""
		} else {
			parentID = payload.Layer.Name
		}
	}
	if IsLocal {
		if err := cleanLocal(); err != nil {
			return err
		}
	}
	return nil
}
Esempio n. 24
0
func getArticle(markdown *mark.MarkDown, fileInfo os.FileInfo) (article Article, err error) {
	setting, ok := markdown.Parts[0].(*mark.Code)
	if !ok {
		err = errors.New("format error")
		return
	}
	err = json.Unmarshal([]byte(html.UnescapeString(setting.Text)), &article)
	if err != nil {
		log.Warningln(err)
		return
	}
	article.JSONFileInfo = &JSONFileInfo{fileInfo.Name(), fileInfo.Size(), fileInfo.Mode(), &JSONTime{fileInfo.ModTime()}, fileInfo.IsDir()}
	return
}
Esempio n. 25
0
func UpdateBuild(config RunnerConfig, id int, state BuildState, trace string) UpdateState {
	request := UpdateBuildRequest{
		Info:  GetRunnerVersion(config),
		Token: config.Token,
		State: state,
		Trace: trace,
	}

	result, statusText := putJSON(getURL(config.URL, "builds/%d.json", id), 200, &request, nil)
	switch result {
	case 200:
		log.Println(config.ShortDescription(), id, "Submitting build to coordinator...", "ok")
		return UpdateSucceeded
	case 404:
		log.Warningln(config.ShortDescription(), id, "Submitting build to coordinator...", "aborted")
		return UpdateAbort
	case 403:
		log.Errorln(config.ShortDescription(), id, "Submitting build to coordinator...", "forbidden")
		return UpdateAbort
	default:
		log.Warningln(config.ShortDescription(), id, "Submitting build to coordinator...", "failed", statusText)
		return UpdateFailed
	}
}
Esempio n. 26
0
func (c *ArchiveCommand) archive() {
	if len(c.files) == 0 {
		logrus.Infoln("No files to archive.")
		return
	}

	logrus.Infoln("Creating archive", filepath.Base(c.Output), "...")
	var files bytes.Buffer
	for _, file := range c.sortedFiles() {
		files.WriteString(string(file) + "\n")
	}

	// create directories to store archive
	os.MkdirAll(filepath.Dir(c.Output), 0700)

	tempFile, err := ioutil.TempFile(filepath.Dir(c.Output), "archive_")
	if err != nil {
		logrus.Fatalln("Failed to create temporary archive", err)
	}
	tempFile.Close()
	defer os.Remove(tempFile.Name())

	logrus.Debugln("Temporary file:", tempFile.Name())

	flags := "-zcPv"
	if c.Silent {
		flags = "-zcP"
	}

	cmd := exec.Command("tar", flags, "-T", "-", "-f", tempFile.Name())
	cmd.Env = os.Environ()
	cmd.Stdin = &files
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	logrus.Debugln("Executing command:", strings.Join(cmd.Args, " "))
	err = cmd.Run()
	if err != nil {
		logrus.Fatalln("Failed to create archive:", err)
	}

	err = os.Rename(tempFile.Name(), c.Output)
	if err != nil {
		logrus.Warningln("Failed to rename archive:", err)
	}

	logrus.Infoln("Done!")
}
func (m *testNetwork) DownloadArtifacts(config common.BuildCredentials, artifactsFile string) common.DownloadState {
	m.downloadCalled++

	if m.downloadState == common.DownloadSucceeded {
		file, err := os.Create(artifactsFile)
		if err != nil {
			logrus.Warningln(err)
			return common.DownloadFailed
		}
		defer file.Close()

		archive := zip.NewWriter(file)
		archive.Create(artifactsTestArchivedFile)
		archive.Close()
	}
	return m.downloadState
}
func (c *CacheExtractorCommand) Execute(context *cli.Context) {
	formatter.SetRunnerFormatter()

	if len(c.File) == 0 {
		logrus.Fatalln("Missing cache file")
	}

	if c.URL != "" {
		err := c.doRetry(c.download)
		if err != nil && !os.IsNotExist(err) {
			logrus.Warningln(err)
		}
	}

	err := archives.ExtractZipFile(c.File)
	if err != nil && !os.IsNotExist(err) {
		logrus.Fatalln(err)
	}
}
Esempio n. 29
0
func (c *ListCommand) Execute(context *cli.Context) {
	err := c.loadConfig()
	if err != nil {
		log.Warningln(err)
		return
	}

	log.WithFields(log.Fields{
		"ConfigFile": c.ConfigFile,
	}).Println("Listing configured runners")

	for _, runner := range c.config.Runners {
		log.WithFields(log.Fields{
			"Executor": runner.RunnerSettings.Executor,
			"Token":    runner.RunnerCredentials.Token,
			"URL":      runner.RunnerCredentials.URL,
		}).Println(runner.Name)
	}
}
func userModeWarning(withRun bool) {
	logrus.WithFields(logrus.Fields{
		"GOOS": runtime.GOOS,
		"uid":  os.Getuid(),
	}).Debugln("Checking runtime mode")

	// everything is supported on windows
	if runtime.GOOS == "windows" {
		return
	}

	systemMode := os.Getuid() == 0

	// We support services on Linux, Windows and Darwin
	noServices :=
		runtime.GOOS != "linux" &&
			runtime.GOOS != "darwin"

	// We don't support services installed as an User on Linux
	noUserService :=
		!systemMode &&
			runtime.GOOS == "linux"

	if systemMode {
		logrus.Infoln("Running in system-mode.")
	} else {
		logrus.Warningln("Running in user-mode.")
	}

	if withRun {
		if noServices {
			logrus.Warningln("You need to manually start builds processing:")
			logrus.Warningln("$ gitlab-runner run")
		} else if noUserService {
			logrus.Warningln("The user-mode requires you to manually start builds processing:")
			logrus.Warningln("$ gitlab-runner run")
		}
	}

	if !systemMode {
		logrus.Warningln("Use sudo for system-mode:")
		logrus.Warningln("$ sudo gitlab-runner...")
	}
	logrus.Infoln("")
}