func (c *ArchiveCommand) processUntracked() {
if !c.Untracked {
return
}
var output bytes.Buffer
cmd := exec.Command("git", "ls-files", "-o")
cmd.Env = os.Environ()
cmd.Stdout = &output
cmd.Stderr = os.Stderr
logrus.Debugln("Executing command:", strings.Join(cmd.Args, " "))
err := cmd.Run()
if err == nil {
reader := bufio.NewReader(&output)
for {
line, _, err := reader.ReadLine()
if err == io.EOF {
break
} else if err != nil {
logrus.Warningln(err)
break
}
c.process(string(line))
}
} else {
logrus.Warningln(err)
}
}
func createZipEntry(archive *zip.Writer, fileName string) error {
fi, err := os.Lstat(fileName)
if err != nil {
logrus.Warningln("File ignored:", err)
return nil
}
fh, err := zip.FileInfoHeader(fi)
if err != nil {
return err
}
fh.Name = fileName
fh.Extra = createZipExtra(fi)
switch fi.Mode() & os.ModeType {
case os.ModeDir:
return createZipDirectoryEntry(archive, fh)
case os.ModeSymlink:
return createZipSymlinkEntry(archive, fh)
case os.ModeNamedPipe, os.ModeSocket, os.ModeDevice:
// Ignore the files that of these types
logrus.Warningln("File ignored:", fileName)
return nil
default:
return createZipFileEntry(archive, fh)
}
}
func waitForInterrupts(finished *bool, abortSignal chan os.Signal, doneSignal chan int) {
signals := make(chan os.Signal)
signal.Notify(signals, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
interrupt := <-signals
if finished != nil {
*finished = true
}
// request stop, but wait for force exit
for interrupt == syscall.SIGQUIT {
log.Warningln("Requested quit, waiting for builds to finish")
interrupt = <-signals
}
log.Warningln("Requested exit:", interrupt)
go func() {
for {
abortSignal <- interrupt
}
}()
select {
case newSignal := <-signals:
log.Fatalln("forced exit:", newSignal)
case <-time.After(common.ShutdownTimeout * time.Second):
log.Fatalln("shutdown timedout")
case <-doneSignal:
}
}
func (n *GitLabClient) UpdateBuild(config RunnerConfig, id int, state BuildState, trace string) UpdateState {
request := UpdateBuildRequest{
Info: n.getRunnerVersion(config),
Token: config.Token,
State: state,
Trace: trace,
}
result, statusText := n.do(config.RunnerCredentials, "PUT", fmt.Sprintf("builds/%d.json", id), 200, &request, nil)
switch result {
case 200:
logrus.Println(config.ShortDescription(), id, "Submitting build to coordinator...", "ok")
return UpdateSucceeded
case 404:
logrus.Warningln(config.ShortDescription(), id, "Submitting build to coordinator...", "aborted")
return UpdateAbort
case 403:
logrus.Errorln(config.ShortDescription(), id, "Submitting build to coordinator...", "forbidden")
return UpdateAbort
case clientError:
logrus.Errorln(config.ShortDescription(), id, "Submitting build to coordinator...", "error", statusText)
return UpdateAbort
default:
logrus.Warningln(config.ShortDescription(), id, "Submitting build to coordinator...", "failed", statusText)
return UpdateFailed
}
}
/*
Install a package with apt-get
*/
func (pkg *Package) installAptPackage(name string) error {
out, err := exec.Command("apt-get", "update").CombinedOutput()
if err != nil {
log.Warningln(string(out))
return err
}
out, err = exec.Command("apt-get", "install", "-y", name).CombinedOutput()
if err != nil {
log.Warningln(string(out))
return err
}
log.Println("Installed Apt package: ", string(out))
return nil
}
func (c *fileArchiver) processPaths() {
for _, path := range c.Paths {
matches, err := filepath.Glob(path)
if err != nil {
logrus.Warningf("%s: %v", path, err)
continue
}
found := 0
for _, match := range matches {
err := filepath.Walk(match, func(path string, info os.FileInfo, err error) error {
if c.process(path) {
found++
}
return nil
})
if err != nil {
logrus.Warningln("Walking", match, err)
}
}
if found == 0 {
logrus.Warningf("%s: no matching files", path)
} else {
logrus.Infof("%s: found %d matching files", path, found)
}
}
}
func dispatchCommand(c *cli.Context, client *etcd.Client, cmd *command.Command) {
targets := c.StringSlice("target")
if targets == nil || len(targets) == 0 {
log.Warningln("no target set! try to send command to all registered host.")
targets = fetchHosts(c, client)
}
if targets == nil {
log.Fatalln("no target to send command.")
} else {
log.Infoln("send command to: ", targets)
}
for _, target := range targets {
key := fmt.Sprintf("/beacon/commands/single/%s/%s/",
target, cmd.Id)
if c.GlobalString("prefix") != "" {
key = fmt.Sprintf("/%s%s", strings.Trim(c.GlobalString("prefix"), "/"), key)
}
if _, err := client.Set(key, cmd.Marshal(), 0); err != nil {
log.WithFields(log.Fields{
"error": err.Error(),
}).Fatalln("send command failed.")
}
}
}
func (c *UnregisterCommand) Execute(context *cli.Context) {
if !common.DeleteRunner(c.URL, c.Token) {
log.Fatalln("Failed to delete runner")
}
err := c.loadConfig()
if err != nil {
log.Warningln(err)
return
}
runners := []*common.RunnerConfig{}
for _, otherRunner := range c.config.Runners {
if otherRunner.RunnerCredentials == c.RunnerCredentials {
continue
}
runners = append(runners, otherRunner)
}
// check if anything changed
if len(c.config.Runners) == len(runners) {
return
}
c.config.Runners = runners
// save config file
err = c.saveConfig()
if err != nil {
log.Fatalln("Failed to update", c.ConfigFile, err)
}
log.Println("Updated", c.ConfigFile)
}
func (n *GitLabClient) GetBuild(config RunnerConfig) (*GetBuildResponse, bool) {
request := GetBuildRequest{
Info: n.getRunnerVersion(config),
Token: config.Token,
}
var response GetBuildResponse
result, statusText := n.do(config.RunnerCredentials, "POST", "builds/register.json", 201, &request, &response)
switch result {
case 201:
logrus.Println(config.ShortDescription(), "Checking for builds...", "received")
return &response, true
case 403:
logrus.Errorln(config.ShortDescription(), "Checking for builds...", "forbidden")
return nil, false
case 404:
logrus.Debugln(config.ShortDescription(), "Checking for builds...", "nothing")
return nil, true
case clientError:
logrus.Errorln(config.ShortDescription(), "Checking for builds...", "error:", statusText)
return nil, false
default:
logrus.Warningln(config.ShortDescription(), "Checking for builds...", "failed:", statusText)
return nil, true
}
}
func (c *ExecCommand) createBuild(repoURL string, abortSignal chan os.Signal) (build *common.Build, err error) {
// Check if we have uncommitted changes
_, err = c.runCommand("git", "diff", "--quiet", "HEAD")
if err != nil {
logrus.Warningln("You most probably have uncommitted changes.")
logrus.Warningln("These changes will not be tested.")
}
// Parse Git settings
sha, err := c.runCommand("git", "rev-parse", "HEAD")
if err != nil {
return
}
beforeSha, err := c.runCommand("git", "rev-parse", "HEAD~1")
if err != nil {
beforeSha = "0000000000000000000000000000000000000000"
}
refName, err := c.runCommand("git", "rev-parse", "--abbrev-ref", "HEAD")
if err != nil {
return
}
build = &common.Build{
GetBuildResponse: common.GetBuildResponse{
ID: 1,
ProjectID: 1,
RepoURL: repoURL,
Commands: "",
Sha: strings.TrimSpace(sha),
RefName: strings.TrimSpace(refName),
BeforeSha: strings.TrimSpace(beforeSha),
AllowGitFetch: false,
Timeout: c.getTimeout(),
Token: "",
Name: "",
Stage: "",
Tag: false,
},
Runner: &common.RunnerConfig{
RunnerSettings: c.RunnerSettings,
},
SystemInterrupt: abortSignal,
}
return
}
func (mr *RunCommand) processRunner(id int, runner *common.RunnerConfig, runners chan *common.RunnerConfig) (err error) {
provider := common.GetExecutor(runner.Executor)
if provider == nil {
return
}
context, err := provider.Acquire(runner)
if err != nil {
log.Warningln("Failed to update executor", runner.Executor, "for", runner.ShortDescription(), err)
return
}
defer provider.Release(runner, context)
// Acquire build slot
if !mr.buildsHelper.acquire(runner) {
return
}
defer mr.buildsHelper.release(runner)
// Receive a new build
buildData, healthy := mr.network.GetBuild(*runner)
mr.makeHealthy(runner.UniqueID(), healthy)
if buildData == nil {
return
}
// Make sure to always close output
buildCredentials := &common.BuildCredentials{
ID: buildData.ID,
Token: buildData.Token,
}
trace := mr.network.ProcessBuild(*runner, buildCredentials)
defer trace.Fail(err)
// Create a new build
build := &common.Build{
GetBuildResponse: *buildData,
Runner: runner,
ExecutorData: context,
SystemInterrupt: mr.abortBuilds,
}
// Add build to list of builds to assign numbers
mr.buildsHelper.addBuild(build)
defer mr.buildsHelper.removeBuild(build)
// Process the same runner by different worker again
// to speed up taking the builds
select {
case runners <- runner:
mr.log().WithField("runner", runner.ShortDescription()).Debugln("Requeued the runner")
default:
mr.log().WithField("runner", runner.ShortDescription()).Debugln("Failed to requeue the runner: ")
}
// Process a build
return build.Run(mr.config, trace)
}
func (c *ArchiveCommand) processPaths() {
for _, path := range c.Paths {
matches, err := filepath.Glob(path)
if err != nil {
logrus.Warningln(err)
continue
}
for _, match := range matches {
err := filepath.Walk(match, func(path string, info os.FileInfo, err error) error {
return c.process(path)
})
if err != nil {
logrus.Warningln("Walking", match, err)
}
}
}
}
func (e *AbstractExecutor) Warningln(args ...interface{}) {
// write to log file
if e.Build != nil {
e.Build.WriteString(helpers.ANSI_BOLD_YELLOW + "WARNING: " + fmt.Sprintln(args...) + helpers.ANSI_RESET)
}
args = append([]interface{}{e.Config.ShortDescription(), e.Build.ID}, args...)
log.Warningln(args...)
}
func getCacheStorageClient(cache *common.CacheConfig) (scl *minio.Client, err error) {
scl, err = minio.New(cache.ServerAddress, cache.AccessKey, cache.SecretKey, cache.Insecure)
if err != nil {
logrus.Warningln(err)
return
}
scl.SetCustomTransport(&bucketLocationTripper{cache.BucketLocation})
return
}
func (m *testNetwork) UploadRawArtifacts(config common.BuildCredentials, reader io.Reader, baseName string, expireIn string) common.UploadState {
m.uploadCalled++
if m.uploadState == common.UploadSucceeded {
var buffer bytes.Buffer
io.Copy(&buffer, reader)
archive, err := zip.NewReader(bytes.NewReader(buffer.Bytes()), int64(buffer.Len()))
if err != nil {
logrus.Warningln(err)
return common.UploadForbidden
}
if len(archive.File) != 1 || archive.File[0].Name != artifactsTestArchivedFile {
logrus.Warningln("Invalid archive:", len(archive.File))
return common.UploadForbidden
}
}
return m.uploadState
}
func (r *retryHelper) doRetry(handler func() (bool, error)) (err error) {
retry, err := handler()
for i := 0; retry && i < r.Retry; i++ {
// wait one second to retry
logrus.Warningln("Retrying...")
time.Sleep(r.RetryTime)
retry, err = handler()
}
return
}
// place a check on the queue
func (sm *Manager) enqueueCheck(check stalker.Check) {
cb, err := json.Marshal(check)
stalker.OnlyLogIf(err)
rconn := sm.rpool.Get()
defer rconn.Close()
res, err := rconn.Do("RPUSH", "worker1", cb)
log.Debugln("Checks now on queue:", res)
if err != nil {
log.Warningln("error pushing check on queue:", err)
}
}
func (c *fileArchiver) isFileChanged(fileName string) bool {
ai, err := os.Stat(fileName)
if ai != nil {
if !c.isChanged(ai.ModTime()) {
return false
}
} else if !os.IsNotExist(err) {
logrus.Warningln(err)
}
return true
}
// Check if pause file exists and sleep until its removed if it does
func (sm *Manager) pauseIfAsked() {
if stalker.FExists(sm.pauseFile) {
log.Warningln("Pausing")
for {
time.Sleep(1 * time.Second)
if !stalker.FExists(sm.pauseFile) {
return
}
}
}
}
func getS3UploadURL(build *common.Build, key string) (url *url.URL) {
cache := build.Runner.Cache
objectName := getCacheObjectName(build, cache, key)
if objectName == "" {
return
}
scl, err := getCacheStorageClient(cache)
if err != nil {
logrus.Warningln(err)
return
}
url, err = scl.PresignedPutObject(cache.BucketName, objectName, time.Second*time.Duration(build.Timeout))
if err != nil {
logrus.Warningln(err)
return
}
return
}
func New(i service.Interface, c *service.Config) (service.Service, error) {
s, err := service.New(i, c)
if err == service.ErrNoServiceSystemDetected {
log.Warningln("No service system detected. Some features may not work!")
return &SimpleService{
i: i,
c: c,
}, nil
}
return s, err
}
func (c *ArchiveCommand) add(path string, info os.FileInfo) (err error) {
if info == nil {
info, err = os.Lstat(path)
}
if err == nil {
c.files[path] = info
} else if os.IsNotExist(err) {
logrus.Warningln("File", path, "doesn't exist")
err = nil
}
return
}
//Push image to Clair for analysis
func Push(image Image) error {
layerCount := len(image.FsLayers)
parentID := ""
if layerCount == 0 {
logrus.Warningln("there is no layer to push")
}
localIP, err := config.LocalServerIP()
if err != nil {
return err
}
hURL := fmt.Sprintf("http://%v/v2", localIP)
if IsLocal {
hURL = strings.Replace(hURL, "/v2", "/local", -1)
logrus.Infof("using %v as local url", hURL)
}
for index, layer := range image.FsLayers {
lUID := xstrings.Substr(layer.BlobSum, 0, 12)
logrus.Infof("Pushing Layer %d/%d [%v]", index+1, layerCount, lUID)
database.InsertRegistryMapping(layer.BlobSum, image.Registry)
payload := v1.LayerEnvelope{Layer: &v1.Layer{
Name: layer.BlobSum,
Path: image.BlobsURI(layer.BlobSum),
ParentName: parentID,
Format: "Docker",
}}
//FIXME Update to TLS
if IsLocal {
payload.Layer.Name = layer.History
payload.Layer.Path += "/layer.tar"
}
payload.Layer.Path = strings.Replace(payload.Layer.Path, image.Registry, hURL, 1)
if err := clair.Push(payload); err != nil {
logrus.Infof("adding layer %d/%d [%v]: %v", index+1, layerCount, lUID, err)
if err != clair.OSNotSupported {
return err
}
parentID = ""
} else {
parentID = payload.Layer.Name
}
}
if IsLocal {
if err := cleanLocal(); err != nil {
return err
}
}
return nil
}
func getArticle(markdown *mark.MarkDown, fileInfo os.FileInfo) (article Article, err error) {
setting, ok := markdown.Parts[0].(*mark.Code)
if !ok {
err = errors.New("format error")
return
}
err = json.Unmarshal([]byte(html.UnescapeString(setting.Text)), &article)
if err != nil {
log.Warningln(err)
return
}
article.JSONFileInfo = &JSONFileInfo{fileInfo.Name(), fileInfo.Size(), fileInfo.Mode(), &JSONTime{fileInfo.ModTime()}, fileInfo.IsDir()}
return
}
func UpdateBuild(config RunnerConfig, id int, state BuildState, trace string) UpdateState {
request := UpdateBuildRequest{
Info: GetRunnerVersion(config),
Token: config.Token,
State: state,
Trace: trace,
}
result, statusText := putJSON(getURL(config.URL, "builds/%d.json", id), 200, &request, nil)
switch result {
case 200:
log.Println(config.ShortDescription(), id, "Submitting build to coordinator...", "ok")
return UpdateSucceeded
case 404:
log.Warningln(config.ShortDescription(), id, "Submitting build to coordinator...", "aborted")
return UpdateAbort
case 403:
log.Errorln(config.ShortDescription(), id, "Submitting build to coordinator...", "forbidden")
return UpdateAbort
default:
log.Warningln(config.ShortDescription(), id, "Submitting build to coordinator...", "failed", statusText)
return UpdateFailed
}
}
func (c *ArchiveCommand) archive() {
if len(c.files) == 0 {
logrus.Infoln("No files to archive.")
return
}
logrus.Infoln("Creating archive", filepath.Base(c.Output), "...")
var files bytes.Buffer
for _, file := range c.sortedFiles() {
files.WriteString(string(file) + "\n")
}
// create directories to store archive
os.MkdirAll(filepath.Dir(c.Output), 0700)
tempFile, err := ioutil.TempFile(filepath.Dir(c.Output), "archive_")
if err != nil {
logrus.Fatalln("Failed to create temporary archive", err)
}
tempFile.Close()
defer os.Remove(tempFile.Name())
logrus.Debugln("Temporary file:", tempFile.Name())
flags := "-zcPv"
if c.Silent {
flags = "-zcP"
}
cmd := exec.Command("tar", flags, "-T", "-", "-f", tempFile.Name())
cmd.Env = os.Environ()
cmd.Stdin = &files
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
logrus.Debugln("Executing command:", strings.Join(cmd.Args, " "))
err = cmd.Run()
if err != nil {
logrus.Fatalln("Failed to create archive:", err)
}
err = os.Rename(tempFile.Name(), c.Output)
if err != nil {
logrus.Warningln("Failed to rename archive:", err)
}
logrus.Infoln("Done!")
}
func (m *testNetwork) DownloadArtifacts(config common.BuildCredentials, artifactsFile string) common.DownloadState {
m.downloadCalled++
if m.downloadState == common.DownloadSucceeded {
file, err := os.Create(artifactsFile)
if err != nil {
logrus.Warningln(err)
return common.DownloadFailed
}
defer file.Close()
archive := zip.NewWriter(file)
archive.Create(artifactsTestArchivedFile)
archive.Close()
}
return m.downloadState
}
func (c *CacheExtractorCommand) Execute(context *cli.Context) {
formatter.SetRunnerFormatter()
if len(c.File) == 0 {
logrus.Fatalln("Missing cache file")
}
if c.URL != "" {
err := c.doRetry(c.download)
if err != nil && !os.IsNotExist(err) {
logrus.Warningln(err)
}
}
err := archives.ExtractZipFile(c.File)
if err != nil && !os.IsNotExist(err) {
logrus.Fatalln(err)
}
}
func (c *ListCommand) Execute(context *cli.Context) {
err := c.loadConfig()
if err != nil {
log.Warningln(err)
return
}
log.WithFields(log.Fields{
"ConfigFile": c.ConfigFile,
}).Println("Listing configured runners")
for _, runner := range c.config.Runners {
log.WithFields(log.Fields{
"Executor": runner.RunnerSettings.Executor,
"Token": runner.RunnerCredentials.Token,
"URL": runner.RunnerCredentials.URL,
}).Println(runner.Name)
}
}
func userModeWarning(withRun bool) {
logrus.WithFields(logrus.Fields{
"GOOS": runtime.GOOS,
"uid": os.Getuid(),
}).Debugln("Checking runtime mode")
// everything is supported on windows
if runtime.GOOS == "windows" {
return
}
systemMode := os.Getuid() == 0
// We support services on Linux, Windows and Darwin
noServices :=
runtime.GOOS != "linux" &&
runtime.GOOS != "darwin"
// We don't support services installed as an User on Linux
noUserService :=
!systemMode &&
runtime.GOOS == "linux"
if systemMode {
logrus.Infoln("Running in system-mode.")
} else {
logrus.Warningln("Running in user-mode.")
}
if withRun {
if noServices {
logrus.Warningln("You need to manually start builds processing:")
logrus.Warningln("$ gitlab-runner run")
} else if noUserService {
logrus.Warningln("The user-mode requires you to manually start builds processing:")
logrus.Warningln("$ gitlab-runner run")
}
}
if !systemMode {
logrus.Warningln("Use sudo for system-mode:")
logrus.Warningln("$ sudo gitlab-runner...")
}
logrus.Infoln("")
}