func (b *Builder) runBuildSetup() error { //TODO REMOVE
if empty, err := common.IsDirEmpty(b.aciHomePath + PATH_RUNLEVELS + PATH_BUILD_SETUP); empty || err != nil {
return nil
}
logs.WithF(b.fields).Info("Running build setup")
for _, e := range manifestApp(b.pod).App.Environment {
logs.WithField("name", e.Name).WithField("value", e.Value).Debug("Adding environment var")
os.Setenv(e.Name, e.Value)
}
logs.WithF(b.fields).Warn("Build setup is deprecated and will be removed. it create unreproductible builds and run as root directly on the host. Please use builder dependencies and builder runlevels instead")
time.Sleep(5 * time.Second)
os.Setenv("BASEDIR", b.aciHomePath)
os.Setenv("TARGET", b.stage2Rootfs+"/..")
os.Setenv("ROOTFS", b.stage2Rootfs+"/../rootfs")
os.Setenv(common.EnvLogLevel, logs.GetLevel().String())
if err := common.ExecCmd(b.stage1Rootfs + PATH_DGR + PATH_BUILDER + "/stage2/build-setup.sh"); err != nil {
return errs.WithEF(err, b.fields, "Build setup failed")
}
return nil
}
func (s *Service) processCheckResult(check Check) {
s.typedCheckersWithStatus[check.Checker] = &check.Status
var combinedStatus error
for _, status := range s.typedCheckersWithStatus {
if status == nil {
logs.WithF(s.fields).Debug("One check have no value, cannot report yet")
return
}
if combinedStatus == nil {
combinedStatus = *status
}
}
if logs.IsDebugEnabled() {
logs.WithF(s.fields.WithField("status", check).WithField("combined", combinedStatus)).Debug("combined status process")
}
if s.currentStatus == nil ||
(*s.currentStatus == nil && combinedStatus != nil) ||
(*s.currentStatus != nil && combinedStatus == nil) {
s.currentStatus = &combinedStatus
s.runNotify()
} else {
logs.WithF(s.fields).Debug("Combined status is same as previous, no report required")
}
}
func (r *RouterHaProxy) isSocketUpdatable(report ServiceReport) bool {
previous := r.lastEvents[report.Service.Name]
if previous == nil || len(previous.Reports) != len(report.Reports) {
logs.WithF(r.RouterCommon.fields.WithField("previous", previous).WithField("current", report)).Debug("Report length is different")
return false
}
for _, new := range report.Reports {
weightOnly := false
for _, old := range previous.Reports {
if new.Host == old.Host &&
new.Port == old.Port &&
new.Name == old.Name &&
new.HaProxyServerOptions == old.HaProxyServerOptions {
weightOnly = true
break
}
}
if !weightOnly {
logs.WithF(r.RouterCommon.fields.WithField("server", new)).Debug("Server was not existing or options has changed")
return false
}
}
return true
}
func (s *Service) runNotify() {
s.runNotifyMutex.Lock()
defer s.runNotifyMutex.Unlock()
s.giveUpWarmup()
if s.currentStatus == nil {
logs.WithF(s.fields).Info("No status to notify")
return
}
if (*s.currentStatus == nil && s.disabled == nil) || s.forceEnable {
logs.WithF(s.fields).Info("Service is available")
if len(s.PreAvailableCommand) > 0 {
if err := ExecCommand(s.PreAvailableCommand, s.PreAvailableMaxDurationInMilli); err != nil {
s.nerve.execFailureCount.WithLabelValues(s.Name, "pre-available", s.Host, strconv.Itoa(s.Port)).Inc()
logs.WithEF(err, s.fields).Warn("Pre available command failed")
}
}
s.warmup()
} else {
if !s.NoMetrics {
s.nerve.availableGauge.WithLabelValues(s.Name, s.Host, strconv.Itoa(s.Port)).Set(0)
}
s.currentWeightIndex = 0
logs.WithEF(*s.currentStatus, s.fields).Warn("Service is not available")
s.reportAndTellIfAtLeastOneReported(true)
}
}
func (aci *Aci) Test() error {
defer aci.giveBackUserRightsToTarget()
hashAcis, err := aci.Install()
if err != nil {
return err
}
logs.WithF(aci.fields).Info("Testing")
ImportInternalTesterIfNeeded(aci.manifest)
logs.WithF(aci.fields).Info("Building test aci")
hashTestAci, err := aci.buildTestAci()
if err != nil {
return err
}
logs.WithF(aci.fields).Info("Running test aci")
if err := aci.runTestAci(hashTestAci, hashAcis); err != nil {
return err
}
logs.WithF(aci.fields).Info("Checking result")
if err := aci.checkResult(); err != nil {
return err
}
return nil
}
// this reuse zk connection if host list is the same
// a new dedicated event chan is created for each call
// zk events are duplicated to all those channels
func NewSharedZkConnection(hosts []string, timeout time.Duration) (*SharedZkConnection, error) {
zkConnectionsMutex.Lock()
defer zkConnectionsMutex.Unlock()
sort.Strings(hosts)
hash := strings.Join(hosts, "")
if _, ok := zkConnections[hash]; !ok {
conn, channel, err := zk.Connect(hosts, timeout)
conn.SetLogger(ZKLogger{})
zkConnections[hash] = &SharedZkConnection{
hash: hash,
Conn: conn,
err: err,
sourceChan: channel,
}
go func(sharedZk *SharedZkConnection) {
events := sharedZk.Subscribe()
connectingCount := 0
for {
select {
case e, ok := <-events:
if !ok {
return
}
if e.Type == zk.EventSession && e.State == zk.StateHasSession {
if sharedZk.connected == false {
logs.WithF(data.WithField("servers", hosts)).Info("Connected to zk")
connectingCount = 0
}
sharedZk.connected = true
} else if (e.Type == zk.EventSession || e.Type == zk.EventType(0)) &&
(e.State == zk.StateDisconnected || e.State == zk.StateExpired) {
if sharedZk.connected == true {
logs.WithF(data.WithField("servers", hosts)).Warn("Connection lost to zk")
connectingCount = 0
}
sharedZk.connected = false
} else if (e.Type == zk.EventSession || e.Type == zk.EventType(0)) &&
(e.State == zk.StateAuthFailed) {
logs.WithF(data.WithField("servers", hosts)).Error("Authentication failure on zk")
} else if (e.Type == zk.EventSession || e.Type == zk.EventType(0)) &&
(e.State == zk.StateConnecting) {
if connectingCount == 1 {
logs.WithF(data.WithField("server", conn.Server())).Warn("Failed to connect to zk. Not reporting nexts servers try until connected")
}
connectingCount++
}
}
}
}(zkConnections[hash])
}
go zkConnections[hash].recipientListPublish()
return zkConnections[hash], zkConnections[hash].err
}
func (hap *HaProxyClient) SocketUpdate() error {
if hap.socketPath == "" {
return errs.WithF(hap.fields, "No socket file specified. Cannot update")
}
logs.WithF(hap.fields).Debug("Updating haproxy by socket")
if err := hap.writeConfig(); err != nil { // just to stay in sync
logs.WithEF(err, hap.fields).Warn("Failed to write configuration file")
}
conn, err := net.Dial("unix", hap.socketPath)
if err != nil {
return errs.WithEF(err, hap.fields.WithField("socket", hap.socketPath), "Failed to connect to haproxy socket")
}
defer conn.Close()
i := 0
b := bytes.Buffer{}
for name, servers := range hap.Backend {
for _, server := range servers {
res := hap.weightRegex.FindStringSubmatch(server)
if len(res) == 3 {
i++
b.WriteString("set weight " + name + "/" + res[1] + " " + res[2] + "\n")
}
}
}
if b.Len() == 0 {
logs.WithF(hap.fields).Debug("Nothing to update by socket. No weight set")
return nil
}
commands := b.Bytes()
logs.WithF(hap.fields.WithField("command", string(commands))).Trace("Running command on hap socket")
count, err := conn.Write(commands)
if count != len(commands) || err != nil {
return errs.WithEF(err, hap.fields.
WithField("written", count).
WithField("len", len(commands)).
WithField("command", string(commands)), "Failed to write command to haproxy")
}
buff := bufio.NewReader(conn)
line, prefix, err := buff.ReadLine()
if err != nil || prefix {
return errs.WithEF(err, hap.fields.WithField("line-too-long", prefix), "Failed to read hap socket response")
}
if string(line) != "" {
return errs.WithF(hap.fields.WithField("response", string(line)), "Bad response for haproxy socket command")
}
return nil
}
func NewAciWithManifest(path string, args BuildArgs, manifestTmpl string, checkWg *sync.WaitGroup) (*Aci, error) {
manifest, err := common.ProcessManifestTemplate(manifestTmpl, nil, false)
if err != nil {
return nil, errs.WithEF(err, data.WithField("content", manifestTmpl), "Failed to process manifest")
}
if manifest.NameAndVersion == "" {
logs.WithField("path", path).Fatal("name is mandatory in manifest")
}
fields := data.WithField("aci", manifest.NameAndVersion.String())
logs.WithF(fields).WithFields(data.Fields{"args": args, "path": path, "manifest": manifest}).Debug("New aci")
fullPath, err := filepath.Abs(path)
if err != nil {
return nil, errs.WithEF(err, fields, "Cannot get fullpath of project")
}
target := fullPath + pathTarget
if Home.Config.TargetWorkDir != "" {
currentAbsDir, err := filepath.Abs(Home.Config.TargetWorkDir + "/" + manifest.NameAndVersion.ShortName())
if err != nil {
return nil, errs.WithEF(err, fields.WithField("path", path), "Invalid target path")
}
target = currentAbsDir
}
aci := &Aci{
fields: fields,
args: args,
path: fullPath,
manifestTmpl: manifestTmpl,
manifest: manifest,
target: target,
FullyResolveDep: true,
checkWg: checkWg,
}
froms, err := manifest.GetFroms()
if err != nil {
logs.WithEF(err, aci.fields).Fatal("Invalid from data")
}
if len(froms) != 0 {
if froms[0].String() == "" {
logs.WithF(aci.fields).Warn("From is deprecated and empty, remove it")
} else {
logs.WithF(aci.fields).Warn("From is deprecated and processed as dependency. move from to dependencies")
aci.manifest.Aci.Dependencies = append(froms, aci.manifest.Aci.Dependencies...)
}
}
return aci, nil
}
func (w *WatcherZookeeper) watchNode(node string, stop <-chan struct{}, doneWaiter *sync.WaitGroup) {
doneWaiter.Add(1)
defer doneWaiter.Done()
fields := w.fields.WithField("node", node)
logs.WithF(fields).Debug("New node watcher")
for {
content, stats, childEvent, err := w.connection.Conn.GetW(node)
if err != nil {
if err == zk.ErrNoNode {
logs.WithEF(err, fields).Warn("Node disappear before watching")
w.reports.removeNode(node)
return
}
w.service.synapse.watcherFailures.WithLabelValues(w.service.Name, PrometheusLabelWatch).Inc()
logs.WithEF(err, fields).Warn("Failed to watch node, retry in 1s")
<-time.After(time.Duration(1000) * time.Millisecond)
if isStopped(stop) {
return
}
continue
}
w.reports.addRawReport(node, content, fields, stats.Ctime)
//if context.oneshot {
// go func() {
// <-rootEvents
// }()
// return
//}
select {
case e := <-childEvent:
logs.WithF(fields.WithField("event", e)).Trace("Receiving event from node")
switch e.Type {
case zk.EventNodeDataChanged | zk.EventNodeCreated | zk.EventNotWatching:
// loop
case zk.EventNodeDeleted:
logs.WithF(fields).Debug("Node deleted")
w.reports.removeNode(node)
return
}
case <-stop:
return
}
}
}
func (aci *Aci) RunBuilderCommand(command common.BuilderCommand) error {
defer aci.giveBackUserRightsToTarget()
logs.WithF(aci.fields).Info("Building")
if err := os.MkdirAll(aci.target, 0777); err != nil {
return errs.WithEF(err, aci.fields, "Cannot create target directory")
}
if err := ioutil.WriteFile(aci.target+common.PathManifestYmlTmpl, []byte(aci.manifestTmpl), 0644); err != nil {
return errs.WithEF(err, aci.fields.WithField("file", aci.target+common.PathManifestYmlTmpl), "Failed to write manifest template")
}
stage1Hash, err := aci.prepareStage1aci()
if err != nil {
return errs.WithEF(err, aci.fields, "Failed to prepare stage1 image")
}
builderHash, err := aci.prepareBuildAci()
if err != nil {
return errs.WithEF(err, aci.fields, "Failed to prepare build image")
}
logs.WithF(aci.fields).Info("Calling rkt to start build")
defer aci.cleanupRun(builderHash, stage1Hash)
if err := Home.Rkt.Run(aci.prepareRktRunArguments(command, builderHash, stage1Hash)); err != nil {
return errs.WithEF(err, aci.fields, "Builder container return with failed status")
}
content, err := common.ExtractManifestContentFromAci(aci.target + pathImageAci)
if err != nil {
logs.WithEF(err, aci.fields).Warn("Failed to write manifest.json")
}
if err := ioutil.WriteFile(aci.target+pathManifestJson, content, 0644); err != nil {
logs.WithEF(err, aci.fields).Warn("Failed to write manifest.json")
}
im := &schema.ImageManifest{}
if err = im.UnmarshalJSON(content); err != nil {
return errs.WithEF(err, aci.fields.WithField("content", string(content)), "Cannot unmarshall json content")
}
fullname := common.ExtractNameVersionFromManifest(im)
logs.WithField("fullname", *fullname).Info("Finished building aci")
if err := ioutil.WriteFile(aci.target+pathVersion, []byte(*fullname), 0644); err != nil {
return errs.WithEF(err, aci.fields, "Failed to write version file in target")
}
return nil
}
func (w *WatcherZookeeper) watchRoot(stop <-chan struct{}, doneWaiter *sync.WaitGroup) {
doneWaiter.Add(1)
defer doneWaiter.Done()
for {
childs, _, rootEvents, err := w.connection.Conn.ChildrenW(w.Path)
if err != nil {
w.service.synapse.watcherFailures.WithLabelValues(w.service.Name, PrometheusLabelWatch).Inc()
logs.WithEF(err, w.fields.WithField("path", w.Path)).Warn("Cannot watch root service path. Retry in 1s")
<-time.After(time.Duration(1000) * time.Millisecond)
if isStopped(stop) {
return
}
continue
}
if len(childs) == 0 {
w.reports.setNoNodes()
} else {
for _, child := range childs {
if _, ok := w.reports.get(w.Path + "/" + child); !ok {
go w.watchNode(w.Path+"/"+child, stop, doneWaiter)
}
}
}
//if context.oneshot {
// go func() {
// <-rootEvents
// }()
// return
//}
select {
case e := <-rootEvents:
logs.WithF(w.fields.WithField("event", e)).Trace("Receiving event for root node")
switch e.Type {
case zk.EventNodeChildrenChanged | zk.EventNodeCreated | zk.EventNodeDataChanged | zk.EventNotWatching:
// loop
case zk.EventNodeDeleted:
logs.WithF(w.fields.WithField("node", w.Path)).Debug("Rootnode deleted")
w.reports.removeAll()
}
case <-stop:
return
}
}
}
func (aci *Aci) Push() error {
defer aci.giveBackUserRightsToTarget()
if Home.Config.Push.Type == "" {
return errs.WithF(aci.fields, "Cannot push, push is not configured in dgr global configuration file")
}
if err := aci.EnsureBuilt(); err != nil {
return err
}
if aci.args.Test {
aci.args.Test = false
if err := aci.Test(); err != nil {
return err
}
}
logs.WithF(aci.fields).Info("Gzipping aci before upload")
im, err := common.ExtractManifestFromAci(aci.target + pathImageAci)
if err != nil {
return errs.WithEF(err, aci.fields.WithField("file", pathImageAci), "Failed to extract manifest from aci file")
}
val, ok := im.Labels.Get("version")
if !ok {
return errs.WithEF(err, aci.fields.WithField("file", pathImageAci), "Failed to get version from aci manifest")
}
if err := aci.zipAci(); err != nil {
return errs.WithEF(err, aci.fields, "Failed to zip aci")
}
logs.WithF(aci.fields).Info("Uploading aci")
if err := common.ExecCmd("curl", "-f", "-i",
"-F", "r=releases",
"-F", "hasPom=false",
"-F", "e=aci",
"-F", "g=com.blablacar.aci.linux.amd64",
"-F", "p=aci",
"-F", "v="+val,
"-F", "a="+strings.Split(string(im.Name), "/")[1],
"-F", "file=@"+aci.target+pathImageGzAci,
"-u", Home.Config.Push.Username+":"+Home.Config.Push.Password,
Home.Config.Push.Url+"/service/local/artifact/maven/content"); err != nil {
return errs.WithEF(err, aci.fields, "Failed to push aci")
}
return nil
}
func (cnt *Aci) checkResult() {
files, err := ioutil.ReadDir(cnt.target + PATH_TESTS + PATH_TARGET + PATH_RESULT)
if err != nil {
panic("Cannot read test result directory" + err.Error())
}
testFound := false
for _, f := range files {
fullPath := cnt.target + PATH_TESTS + PATH_TARGET + PATH_RESULT + "/" + f.Name()
content, err := ioutil.ReadFile(fullPath)
if err != nil {
panic("Cannot read result file" + f.Name() + err.Error())
}
if !strings.HasSuffix(f.Name(), STATUS_SUFFIX) {
if testFound == false && string(content) != "1..0\n" {
testFound = true
}
continue
}
if string(content) != "0\n" {
logs.WithF(cnt.fields).WithField("file", f.Name()).Error("Failed test")
os.Exit(2)
}
}
if cnt.args.NoTestFail && !testFound {
panic("No tests found")
}
}
func Logger() macaron.Handler {
var reqCounter int64
return func(ctx *macaron.Context, log *log.Logger) {
start := time.Now()
fields := data.WithField("method", ctx.Req.Method).
WithField("uri", ctx.Req.RequestURI).
WithField("ip", ctx.RemoteAddr()).
WithField("id", atomic.AddInt64(&reqCounter, 1))
if logs.IsDebugEnabled() {
logs.WithF(fields).Trace("Request received")
}
rw := ctx.Resp.(macaron.ResponseWriter)
ctx.Next()
if logs.IsInfoEnabled() {
fields = fields.WithField("duration", time.Since(start)).WithField("status", rw.Status())
var lvl logs.Level
if rw.Status() >= 500 && rw.Status() < 600 {
lvl = logs.ERROR
} else {
lvl = logs.DEBUG
}
logs.LogEntry(&logs.Entry{
Fields: fields,
Level: lvl,
Message: "Request completed",
})
}
}
}
func (c *CheckCommon) CommonRun(checker Checker, statusChange chan<- Check, stop <-chan struct{}, doneWait *sync.WaitGroup) {
logs.WithF(c.fields).Info("Starting check")
doneWait.Add(1)
defer doneWait.Done()
for {
status := checker.Check()
if logs.IsTraceEnabled() {
logs.WithEF(status, c.fields).Trace("Check done")
}
if status != nil {
logs.WithEF(status, c.fields).Debug("Failed check")
}
if status != nil && !c.service.NoMetrics {
c.service.nerve.checkerFailureCount.WithLabelValues(c.service.Name, c.Host, strconv.Itoa(c.Port), c.Type).Inc()
}
c.saveStatus(status)
current := c.stableStatus
latest := c.latestStatuses
if (latest[0] == nil && sameLastStatusCount(latest) >= c.Rise && (current == nil || *current != nil)) ||
(latest[0] != nil && sameLastStatusCount(latest) >= c.Fall && (current == nil || *current == nil)) {
c.stableStatus = &status
statusChange <- Check{checker, *c.stableStatus}
}
select {
case <-stop:
logs.WithFields(c.fields).Debug("Stopping check")
return
case <-time.After(time.Duration(c.CheckIntervalInMilli) * time.Millisecond):
}
}
}
func (b *Builder) Build() error {
logs.WithF(b.fields).Info("Building aci")
lfd, err := rktcommon.GetRktLockFD()
if err != nil {
return errs.WithEF(err, b.fields, "can't get rkt lock fd")
}
if err := sys.CloseOnExec(lfd, true); err != nil {
return errs.WithEF(err, b.fields, "can't set FD_CLOEXEC on rkt lock")
}
if err := b.runBuild(); err != nil {
return err
}
if err := b.writeManifest(); err != nil {
return err
}
if err := b.tarAci(); err != nil {
return err
}
return nil
}
func (aci *Aci) prepareBuildAci() (string, error) {
logs.WithFields(aci.fields).Debug("Preparing builder")
if err := os.MkdirAll(aci.target+pathBuilder+common.PathRootfs, 0777); err != nil {
return "", errs.WithEF(err, aci.fields.WithField("path", aci.target+pathBuilder), "Failed to create builder aci path")
}
if err := ioutil.WriteFile(aci.target+pathBuilder+common.PathRootfs+"/.keep", []byte(""), 0644); err != nil {
return "", errs.WithEF(err, aci.fields.WithField("file", aci.target+pathBuilder+common.PathRootfs+"/.keep"), "Failed to write keep file")
}
capa, err := types.NewLinuxCapabilitiesRetainSet("all")
if err != nil {
return "", errs.WithEF(err, aci.fields, "Failed to create all capability retain Set")
}
allIsolator, err := capa.AsIsolator()
if err != nil {
return "", errs.WithEF(err, aci.fields, "Failed to prepare all retain set isolator")
}
aci.manifest.Aci.App.Isolators = types.Isolators([]types.Isolator{*allIsolator})
if err := common.WriteAciManifest(aci.manifest, aci.target+pathBuilder+common.PathManifest, common.PrefixBuilder+aci.manifest.NameAndVersion.Name(), BuildVersion); err != nil {
return "", err
}
if err := aci.tarAci(aci.target + pathBuilder); err != nil {
return "", err
}
logs.WithF(aci.fields.WithField("path", aci.target+pathBuilder+pathImageAci)).Info("Importing build to rkt")
hash, err := Home.Rkt.FetchInsecure(aci.target + pathBuilder + pathImageAci)
if err != nil {
return "", errs.WithEF(err, aci.fields, "fetch of builder aci failed")
}
return hash, nil
}
func (e Env) runFleetCmdInternal(getOutput bool, args []string) (string, string, error) {
logs.WithF(e.fields).WithField("command", strings.Join(args, " ")).Debug("Running command on fleet")
if e.config.Fleet.Endpoint == "" {
return "", "", errors.New("Cannot find fleet.endpoint env config to call fleetctl")
}
envs := map[string]string{}
envs[FLEETCTL_ENDPOINT] = e.config.Fleet.Endpoint
if e.config.Fleet.Username != "" {
envs[FLEETCTL_SSH_USERNAME] = e.config.Fleet.Username
}
envs[FLEETCTL_STRICT_HOST_KEY_CHECKING] = fmt.Sprintf("%t", e.config.Fleet.Strict_host_key_checking)
envs[FLEETCTL_SUDO] = fmt.Sprintf("%t", e.config.Fleet.Sudo)
args = append([]string{"fleetctl"}, args...)
for key, val := range envs {
args = append([]string{key + "='" + val + "'"}, args...)
}
var stdout string
var stderr string
var err error
if getOutput {
stdout, stderr, err = common.ExecCmdGetStdoutAndStderr("bash", "-c", strings.Join(args, " "))
} else {
err = common.ExecCmd("bash", "-c", strings.Join(args, " "))
}
return stdout, stderr, err
}
func (b *Builder) runBuild() error {
command, err := b.getCommandPath()
if err != nil {
return err
}
logs.WithF(b.fields).Debug("Running build command")
args, env, err := b.prepareNspawnArgsAndEnv(command)
if err != nil {
return err
}
os.Remove(b.stage1Rootfs + "/etc/machine-id")
if logs.IsDebugEnabled() {
logs.WithField("command", strings.Join([]string{args[0], " ", strings.Join(args[1:], " ")}, " ")).Debug("Running external command")
}
// var stderr bytes.Buffer
cmd := exec.Command(args[0], args[1:]...)
cmd.Env = env
cmd.Stdout = os.Stdout
cmd.Stdin = os.Stdin
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return errs.WithEF(err, b.fields, "Builder run failed")
}
return nil
}
func (aci *Aci) Clean() {
logs.WithF(aci.fields).Debug("Cleaning")
if err := os.RemoveAll(aci.target + "/"); err != nil {
logs.WithEF(err, aci.fields).WithField("dir", aci.target).Warn("Cannot remove directory")
}
}
func (p *Pod) Push() {
logs.WithF(p.fields).Info("Pushing")
p.Build()
checkVersion := make(chan bool, 1)
for _, e := range p.manifest.Pod.Apps {
aci, err := NewAciWithManifest(p.path+"/"+e.Name, p.args, p.toAciManifest(e), &checkVersion)
if err != nil {
logs.WithEF(err, p.fields.WithField("name", e.Name)).Fatal("Cannot prepare aci")
}
aci.podName = &p.manifest.Name
aci.Push()
}
for range p.manifest.Pod.Apps {
<-checkVersion
}
if err := utils.ExecCmd("curl", "-i",
"-F", "r=releases",
"-F", "hasPom=false",
"-F", "e=pod",
"-F", "g=com.blablacar.aci.linux.amd64",
"-F", "p=pod",
"-F", "v="+p.manifest.Name.Version(),
"-F", "a="+p.manifest.Name.ShortName(),
"-F", "file=@"+p.target+"/pod-manifest.json",
"-u", cnt.Home.Config.Push.Username+":"+cnt.Home.Config.Push.Password,
cnt.Home.Config.Push.Url+"/service/local/artifact/maven/content"); err != nil {
logs.WithEF(err, p.fields).Fatal("Cannot push pod")
}
}
func (aci *Aci) prepareBuildAci() (string, error) {
logs.WithFields(aci.fields).Debug("Preparing builder")
if err := os.MkdirAll(aci.target+pathBuilder+common.PathRootfs, 0777); err != nil {
return "", errs.WithEF(err, aci.fields.WithField("path", aci.target+pathBuilder), "Failed to create builder aci path")
}
if err := ioutil.WriteFile(aci.target+pathBuilder+common.PathRootfs+"/.keep", []byte(""), 0644); err != nil {
return "", errs.WithEF(err, aci.fields.WithField("file", aci.target+pathBuilder+common.PathRootfs+"/.keep"), "Failed to write keep file")
}
if err := common.WriteAciManifest(aci.manifest, aci.target+pathBuilder+common.PathManifest, common.PrefixBuilder+aci.manifest.NameAndVersion.Name(), dgrVersion); err != nil {
return "", err
}
if err := aci.tarAci(aci.target + pathBuilder); err != nil {
return "", err
}
logs.WithF(aci.fields.WithField("path", aci.target+pathBuilder+pathImageAci)).Info("Importing build to rkt")
hash, err := Home.Rkt.Fetch(aci.target + pathBuilder + pathImageAci)
if err != nil {
return "", errs.WithEF(err, aci.fields, "fetch of builder aci failed")
}
return hash, nil
}
func (aci *Aci) copyInternals() {
logs.WithF(aci.fields).Debug("Copy internals")
os.MkdirAll(aci.rootfs+PATH_CNT+PATH_BIN, 0755)
os.MkdirAll(aci.rootfs+"/bin", 0755) // this is required or systemd-nspawn will create symlink on it
os.MkdirAll(aci.rootfs+"/usr/bin", 0755) // this is required by systemd-nspawn
busybox, _ := dist.Asset("dist/bindata/busybox")
if err := ioutil.WriteFile(aci.rootfs+PATH_CNT+PATH_BIN+"/busybox", busybox, 0777); err != nil {
panic(err)
}
confd, _ := dist.Asset("dist/bindata/confd")
if err := ioutil.WriteFile(aci.rootfs+PATH_CNT+PATH_BIN+"/confd", confd, 0777); err != nil {
panic(err)
}
attributeMerger, _ := dist.Asset("dist/bindata/attributes-merger")
if err := ioutil.WriteFile(aci.rootfs+PATH_CNT+PATH_BIN+"/attributes-merger", attributeMerger, 0777); err != nil {
panic(err)
}
confdFile := `backend = "env"
confdir = "/cnt"
prefix = "/confd"
log-level = "debug"
`
os.MkdirAll(aci.rootfs+PATH_CNT+"/prestart", 0755)
if err := ioutil.WriteFile(aci.rootfs+PATH_CNT+"/prestart/confd.toml", []byte(confdFile), 0777); err != nil {
panic(err)
}
if err := ioutil.WriteFile(aci.rootfs+PATH_CNT+PATH_BIN+"/prestart", []byte(PRESTART), 0777); err != nil {
panic(err)
}
}
func (aci *Aci) upload(name *common.ACFullname) error {
if Home.Config.Push.Type == "maven" && name.DomainName() == "aci.blbl.cr" { // TODO this definitely need to be removed
logs.WithF(aci.fields).Info("Uploading aci")
if err := common.ExecCmd("curl", "-f", "-i",
"-F", "r=releases",
"-F", "hasPom=false",
"-F", "e=aci",
"-F", "g=com.blablacar.aci.linux.amd64",
"-F", "p=aci",
"-F", "v="+name.Version(),
"-F", "a="+strings.Split(string(name.Name()), "/")[1],
"-F", "file=@"+aci.target+pathImageGzAci,
"-u", Home.Config.Push.Username+":"+Home.Config.Push.Password,
Home.Config.Push.Url+"/service/local/artifact/maven/content"); err != nil {
return errs.WithEF(err, aci.fields, "Failed to push aci")
}
} else {
systemConf := Home.Config.Rkt.SystemConfig
if systemConf == "" {
systemConf = "/usr/lib/rkt"
}
localConf := Home.Config.Rkt.LocalConfig
if localConf == "" {
localConf = "/etc/rkt"
}
conf, err := config.GetConfigFrom(systemConf, localConf)
if err != nil {
return errs.WithEF(err, aci.fields, "Failed to get rkt configuration")
}
upload := Uploader{
Acipath: aci.target + pathImageGzAci,
Ascpath: aci.target + pathImageGzAciAsc,
Uri: aci.manifest.NameAndVersion.String(),
Debug: false,
SetHTTPHeaders: func(r *http.Request) {
if r.URL == nil {
return
}
headerer, ok := conf.AuthPerHost[r.URL.Host]
if !ok {
logs.WithFields(aci.fields).WithField("domain", r.URL.Host).
Warn("No auth credential found in rkt configuration for this domain")
return
}
header := headerer.Header()
for k, v := range header {
r.Header[k] = append(r.Header[k], v...)
}
},
}
err = upload.Upload()
if err != nil {
return errs.WithEF(err, aci.fields, "Failed to upload aci")
}
}
return nil
}
func NewAciWithManifest(path string, args BuildArgs, manifest spec.AciManifest, checked *chan bool) (*Aci, error) {
if manifest.NameAndVersion == "" {
logs.WithField("path", path).Fatal("name is mandatory in manifest")
}
fields := data.WithField("aci", manifest.NameAndVersion.String())
logs.WithF(fields).WithFields(data.Fields{"args": args, "path": path, "manifest": manifest}).Debug("New aci")
fullPath, err := filepath.Abs(path)
if err != nil {
return nil, errs.WithEF(err, fields, "Cannot get fullpath of project")
}
target := fullPath + PATH_TARGET
if cnt.Home.Config.TargetWorkDir != "" {
currentAbsDir, err := filepath.Abs(cnt.Home.Config.TargetWorkDir + "/" + manifest.NameAndVersion.ShortName())
if err != nil {
return nil, errs.WithEF(err, fields.WithField("path", path), "Invalid target path")
}
target = currentAbsDir
}
aci := &Aci{
fields: fields,
args: args,
path: fullPath,
manifest: manifest,
target: target,
rootfs: target + PATH_ROOTFS,
FullyResolveDep: true,
}
go aci.checkLatestVersions(checked)
return aci, nil
}
func (aci *Aci) prepareBuildAci() (string, error) {
logs.WithFields(aci.fields).Debug("Preparing builder")
if err := os.MkdirAll(aci.target+pathBuilder+common.PathRootfs, 0777); err != nil {
return "", errs.WithEF(err, aci.fields.WithField("path", aci.target+pathBuilder), "Failed to create builder aci path")
}
if err := ioutil.WriteFile(aci.target+pathBuilder+common.PathRootfs+"/.keep", []byte(""), 0644); err != nil {
return "", errs.WithEF(err, aci.fields.WithField("file", aci.target+pathBuilder+common.PathRootfs+"/.keep"), "Failed to write keep file")
}
aci.manifest.Aci.App.Isolators = []common.Isolator{{Name: "os/linux/capabilities-retain-set", Value: common.LinuxCapabilitiesSetValue{Set: []types.LinuxCapability{"all"}}}}
if err := common.WriteAciManifest(aci.manifest, aci.target+pathBuilder+common.PathManifest, common.PrefixBuilder+aci.manifest.NameAndVersion.Name(), dgrVersion); err != nil {
return "", err
}
if err := aci.tarAci(aci.target + pathBuilder); err != nil {
return "", err
}
logs.WithF(aci.fields.WithField("path", aci.target+pathBuilder+pathImageAci)).Info("Importing build to rkt")
hash, err := Home.Rkt.FetchInsecure(aci.target + pathBuilder + pathImageAci)
if err != nil {
return "", errs.WithEF(err, aci.fields, "fetch of builder aci failed")
}
return hash, nil
}
func ExecCommandFull(cmd []string, env []string, timeoutInMilli int) error {
command := exec.Command(cmd[0], cmd[1:]...)
var b bytes.Buffer
command.Stdout = &b
command.Stderr = &b
command.Env = env
if err := command.Start(); err != nil {
return errs.WithEF(err, data.WithField("cmd", cmd), "Failed to start command")
}
var after *errs.EntryError
timer := time.AfterFunc(time.Duration(timeoutInMilli)*time.Millisecond, func() {
data := data.WithField("command", strings.Join(cmd, " ")).WithField("timeout", timeoutInMilli)
logs.WithF(data).Debug("Command timeout")
after = errs.WithF(data, "Exec command timeout")
command.Process.Kill()
})
err := command.Wait()
timer.Stop()
if logs.IsTraceEnabled() {
logs.WithField("cmd", cmd).WithField("output", string(b.Bytes())).Trace("Command output")
}
if err != nil {
return errs.WithEF(err, data.WithField("cmd", cmd).
WithField("output", string(b.Bytes())), "Command failed").
WithErr(after)
}
return nil
}
func (s *Service) Start(stopper <-chan struct{}, stopWait *sync.WaitGroup) {
logs.WithFields(s.fields).Info("Starting service check")
stopWait.Add(1)
defer stopWait.Done()
checkStopWait := &sync.WaitGroup{}
statusChange := make(chan Check, 2)
for checker := range s.typedCheckersWithStatus {
go checker.Run(statusChange, stopper, checkStopWait)
}
for {
select {
case status := <-statusChange:
logs.WithF(s.fields.WithField("status", status)).Debug("New status received")
s.processCheckResult(status)
case <-stopper: //TODO since stop is the same everywhere, statusChange chan may stay stuck on shutdown
logs.WithFields(s.fields).Debug("Stop requested")
checkStopWait.Wait()
close(statusChange)
if *s.SetServiceAsDownOnShutdown {
wait := &sync.WaitGroup{}
wait.Add(1)
s.Disable(wait, false)
wait.Wait()
}
for reporter := range s.typedReportersWithReported {
reporter.Destroy()
}
return
case <-time.After(time.Duration(s.ReportReplayInMilli) * time.Millisecond):
s.reportAndTellIfAtLeastOneReported(false)
}
}
}
func (p *Pod) Graph() {
logs.WithF(p.fields).Info("Graphing")
os.MkdirAll(p.target, 0777)
var buffer bytes.Buffer
buffer.WriteString("digraph {\n")
buffer.WriteString(" {\n")
buffer.WriteString(" ")
buffer.WriteString("\"")
buffer.WriteString(p.manifest.Name.ShortNameId())
buffer.WriteString("\"")
buffer.WriteString(" [style=filled, fillcolor=yellow, shape=box]\n")
buffer.WriteString(" }\n")
for _, e := range p.manifest.Pod.Apps {
for _, d := range e.Dependencies {
buffer.WriteString(" ")
buffer.WriteString("\"")
buffer.WriteString(d.ShortNameId())
buffer.WriteString("\"")
buffer.WriteString(" -> ")
buffer.WriteString("\"")
buffer.WriteString(p.manifest.Name.ShortNameId())
buffer.WriteString("\"")
buffer.WriteString("\n")
}
}
buffer.WriteString("}\n")
ioutil.WriteFile(p.target+"/graph.dot", buffer.Bytes(), 0644)
}
func (p *Pod) Install() ([]string, error) {
logs.WithF(p.fields).Info("Installing")
hashs := []string{}
if err := p.CleanAndBuild(); err != nil {
return hashs, err
}
for _, e := range p.manifest.Pod.Apps {
tmpl, err := p.toAciManifestTemplate(e)
if err != nil {
return nil, err
}
aci, err := NewAciWithManifest(p.path+"/"+e.Name, p.args, tmpl, p.checkWg)
if err != nil {
logs.WithEF(err, p.fields.WithField("name", e.Name)).Fatal("Cannot prepare aci")
}
aci.podName = &p.manifest.Name
hash, err := aci.Install()
if err != nil {
return hashs, err
}
hashs = append(hashs, hash...)
}
return hashs, nil
}