func (mr *RunCommand) feedRunner(runner *common.RunnerConfig, runners chan *common.RunnerConfig) {
if !mr.isHealthy(runner.UniqueID()) {
return
}
runners <- runner
}
func (n *GitLabClient) GetBuild(config common.RunnerConfig) (*common.GetBuildResponse, bool) {
request := common.GetBuildRequest{
Info: n.getRunnerVersion(config),
Token: config.Token,
}
var response common.GetBuildResponse
result, statusText, certificates := n.doJSON(config.RunnerCredentials, "POST", "builds/register.json", 201, &request, &response)
switch result {
case 201:
config.Log().WithFields(logrus.Fields{
"build": strconv.Itoa(response.ID),
"repo_url": response.RepoCleanURL(),
}).Println("Checking for builds...", "received")
response.TLSCAChain = certificates
return &response, true
case 403:
config.Log().Errorln("Checking for builds...", "forbidden")
return nil, false
case 204, 404:
config.Log().Debugln("Checking for builds...", "nothing")
return nil, true
case clientError:
config.Log().WithField("status", statusText).Errorln("Checking for builds...", "error")
return nil, false
default:
config.Log().WithField("status", statusText).Warningln("Checking for builds...", "failed")
return nil, true
}
}
func (n *GitLabClient) UpdateBuild(config common.RunnerConfig, id int, state common.BuildState, trace *string) common.UpdateState {
request := common.UpdateBuildRequest{
Info: n.getRunnerVersion(config),
Token: config.Token,
State: state,
Trace: trace,
}
log := config.Log().WithField("build", id)
result, statusText, _ := n.doJSON(config.RunnerCredentials, "PUT", fmt.Sprintf("builds/%d.json", id), 200, &request, nil)
switch result {
case 200:
log.Debugln("Submitting build to coordinator...", "ok")
return common.UpdateSucceeded
case 404:
log.Warningln("Submitting build to coordinator...", "aborted")
return common.UpdateAbort
case 403:
log.WithField("status", statusText).Errorln("Submitting build to coordinator...", "forbidden")
return common.UpdateAbort
case clientError:
log.WithField("status", statusText).Errorln("Submitting build to coordinator...", "error")
return common.UpdateAbort
default:
log.WithField("status", statusText).Warningln("Submitting build to coordinator...", "failed")
return common.UpdateFailed
}
}
func (c *clientBuildTrace) resendPatch(id int, config common.RunnerConfig, buildCredentials *common.BuildCredentials, tracePatch common.BuildTracePatch) (update common.UpdateState) {
config.Log().Warningln(id, "Resending trace patch due to range missmatch")
update = c.client.PatchTrace(config, buildCredentials, tracePatch)
if update == common.UpdateRangeMissmatch {
config.Log().Errorln(id, "Appending trace to coordinator...", "failed due to range missmatch")
update = common.UpdateFailed
}
return
}
func (m *machineProvider) updateMachines(machines []string, config *common.RunnerConfig) (data machinesData) {
data.Runner = config.ShortDescription()
for _, name := range machines {
details := m.machineDetails(name, false)
err := m.updateMachine(config, &data, details)
if err != nil {
m.remove(details.Name, err)
}
data.Add(details.State)
}
return
}
func (mr *RunCommand) processRunner(id int, runner *common.RunnerConfig, runners chan *common.RunnerConfig) (err error) {
provider := common.GetExecutor(runner.Executor)
if provider == nil {
return
}
context, err := provider.Acquire(runner)
if err != nil {
log.Warningln("Failed to update executor", runner.Executor, "for", runner.ShortDescription(), err)
return
}
defer provider.Release(runner, context)
// Acquire build slot
if !mr.buildsHelper.acquire(runner) {
return
}
defer mr.buildsHelper.release(runner)
// Receive a new build
buildData, healthy := mr.network.GetBuild(*runner)
mr.makeHealthy(runner.UniqueID(), healthy)
if buildData == nil {
return
}
// Make sure to always close output
buildCredentials := &common.BuildCredentials{
ID: buildData.ID,
Token: buildData.Token,
}
trace := mr.network.ProcessBuild(*runner, buildCredentials)
defer trace.Fail(err)
// Create a new build
build := &common.Build{
GetBuildResponse: *buildData,
Runner: runner,
ExecutorData: context,
SystemInterrupt: mr.abortBuilds,
}
// Add build to list of builds to assign numbers
mr.buildsHelper.addBuild(build)
defer mr.buildsHelper.removeBuild(build)
// Process the same runner by different worker again
// to speed up taking the builds
select {
case runners <- runner:
mr.log().WithField("runner", runner.ShortDescription()).Debugln("Requeued the runner")
default:
mr.log().WithField("runner", runner.ShortDescription()).Debugln("Failed to requeue the runner: ")
}
// Process a build
return build.Run(mr.config, trace)
}
func (m *machineProvider) Acquire(config *common.RunnerConfig) (data common.ExecutorData, err error) {
if config.Machine == nil || config.Machine.MachineName == "" {
err = fmt.Errorf("Missing Machine options")
return
}
machines, err := m.loadMachines(config)
if err != nil {
return
}
// Lock updating machines, because two Acquires can be run at the same time
m.acquireLock.Lock()
// Update a list of currently configured machines
machinesData := m.updateMachines(machines, config)
// Pre-create machines
m.createMachines(config, &machinesData)
m.acquireLock.Unlock()
logrus.WithFields(machinesData.Fields()).
WithField("runner", config.ShortDescription()).
WithField("minIdleCount", config.Machine.IdleCount).
WithField("maxMachines", config.Limit).
WithField("time", time.Now()).
Debugln("Docker Machine Details")
machinesData.writeDebugInformation()
// Try to find a free machine
details := m.findFreeMachine(machines...)
if details != nil {
data = details
return
}
// If we have a free machines we can process a build
if config.Machine.IdleCount != 0 && machinesData.Idle == 0 {
err = errors.New("No free machines that can process builds")
}
return
}
func (n *GitLabClient) PatchTrace(config common.RunnerConfig, buildCredentials *common.BuildCredentials, tracePatch common.BuildTracePatch) common.UpdateState {
id := buildCredentials.ID
contentRange := fmt.Sprintf("%d-%d", tracePatch.Offset(), tracePatch.Limit())
headers := make(http.Header)
headers.Set("Content-Range", contentRange)
headers.Set("BUILD-TOKEN", buildCredentials.Token)
uri := fmt.Sprintf("builds/%d/trace.txt", id)
request := bytes.NewReader(tracePatch.Patch())
response, err := n.doRaw(config.RunnerCredentials, "PATCH", uri, request, "text/plain", headers)
if err != nil {
config.Log().Errorln("Appending trace to coordinator...", "error", err.Error())
return common.UpdateFailed
}
defer response.Body.Close()
defer io.Copy(ioutil.Discard, response.Body)
remoteState := response.Header.Get("Build-Status")
remoteRange := response.Header.Get("Range")
log := config.Log().WithFields(logrus.Fields{
"build": id,
"sent-log": contentRange,
"build-log": remoteRange,
"build-status": remoteState,
"code": response.StatusCode,
"status": response.Status,
})
if remoteState == "canceled" {
log.Warningln("Appending trace to coordinator", "aborted")
return common.UpdateAbort
}
switch response.StatusCode {
case 202:
log.Debugln("Appending trace to coordinator...", "ok")
return common.UpdateSucceeded
case 404:
log.Warningln("Appending trace to coordinator...", "not-found")
return common.UpdateNotFound
case 406:
log.Errorln("Appending trace to coordinator...", "forbidden")
return common.UpdateAbort
case 416:
log.Warningln("Appending trace to coordinator...", "range missmatch")
remoteRange := strings.Split(remoteRange, "-")
newOffset, _ := strconv.Atoi(remoteRange[1])
tracePatch.SetNewOffset(newOffset)
return common.UpdateRangeMissmatch
case clientError:
log.Errorln("Appending trace to coordinator...", "error")
return common.UpdateAbort
default:
log.Warningln("Appending trace to coordinator...", "failed")
return common.UpdateFailed
}
}
func machineFilter(config *common.RunnerConfig) string {
return machineFormat(config.ShortDescription(), config.Machine.MachineName)
}