Example #1
0
func (b *Builder) runBuildSetup() error { //TODO REMOVE
	if empty, err := common.IsDirEmpty(b.aciHomePath + PATH_RUNLEVELS + PATH_BUILD_SETUP); empty || err != nil {
		return nil
	}

	logs.WithF(b.fields).Info("Running build setup")

	for _, e := range manifestApp(b.pod).App.Environment {
		logs.WithField("name", e.Name).WithField("value", e.Value).Debug("Adding environment var")
		os.Setenv(e.Name, e.Value)
	}

	logs.WithF(b.fields).Warn("Build setup is deprecated and will be removed. it create unreproductible builds and run as root directly on the host. Please use builder dependencies and builder runlevels instead")
	time.Sleep(5 * time.Second)

	os.Setenv("BASEDIR", b.aciHomePath)
	os.Setenv("TARGET", b.stage2Rootfs+"/..")
	os.Setenv("ROOTFS", b.stage2Rootfs+"/../rootfs")
	os.Setenv(common.EnvLogLevel, logs.GetLevel().String())

	if err := common.ExecCmd(b.stage1Rootfs + PATH_DGR + PATH_BUILDER + "/stage2/build-setup.sh"); err != nil {
		return errs.WithEF(err, b.fields, "Build setup failed")
	}

	return nil
}
Example #2
0
func (s *Service) processCheckResult(check Check) {
	s.typedCheckersWithStatus[check.Checker] = &check.Status
	var combinedStatus error
	for _, status := range s.typedCheckersWithStatus {
		if status == nil {
			logs.WithF(s.fields).Debug("One check have no value, cannot report yet")
			return
		}
		if combinedStatus == nil {
			combinedStatus = *status
		}
	}

	if logs.IsDebugEnabled() {
		logs.WithF(s.fields.WithField("status", check).WithField("combined", combinedStatus)).Debug("combined status process")
	}

	if s.currentStatus == nil ||
		(*s.currentStatus == nil && combinedStatus != nil) ||
		(*s.currentStatus != nil && combinedStatus == nil) {
		s.currentStatus = &combinedStatus
		s.runNotify()
	} else {
		logs.WithF(s.fields).Debug("Combined status is same as previous, no report required")
	}
}
Example #3
0
func (r *RouterHaProxy) isSocketUpdatable(report ServiceReport) bool {
	previous := r.lastEvents[report.Service.Name]

	if previous == nil || len(previous.Reports) != len(report.Reports) {
		logs.WithF(r.RouterCommon.fields.WithField("previous", previous).WithField("current", report)).Debug("Report length is different")
		return false
	}

	for _, new := range report.Reports {
		weightOnly := false
		for _, old := range previous.Reports {
			if new.Host == old.Host &&
				new.Port == old.Port &&
				new.Name == old.Name &&
				new.HaProxyServerOptions == old.HaProxyServerOptions {
				weightOnly = true
				break
			}
		}

		if !weightOnly {

			logs.WithF(r.RouterCommon.fields.WithField("server", new)).Debug("Server was not existing or options has changed")
			return false
		}
	}
	return true
}
Example #4
0
func (s *Service) runNotify() {
	s.runNotifyMutex.Lock()
	defer s.runNotifyMutex.Unlock()

	s.giveUpWarmup()
	if s.currentStatus == nil {
		logs.WithF(s.fields).Info("No status to notify")
		return
	}

	if (*s.currentStatus == nil && s.disabled == nil) || s.forceEnable {
		logs.WithF(s.fields).Info("Service is available")

		if len(s.PreAvailableCommand) > 0 {
			if err := ExecCommand(s.PreAvailableCommand, s.PreAvailableMaxDurationInMilli); err != nil {
				s.nerve.execFailureCount.WithLabelValues(s.Name, "pre-available", s.Host, strconv.Itoa(s.Port)).Inc()
				logs.WithEF(err, s.fields).Warn("Pre available command failed")
			}
		}

		s.warmup()
	} else {
		if !s.NoMetrics {
			s.nerve.availableGauge.WithLabelValues(s.Name, s.Host, strconv.Itoa(s.Port)).Set(0)
		}
		s.currentWeightIndex = 0
		logs.WithEF(*s.currentStatus, s.fields).Warn("Service is not available")
		s.reportAndTellIfAtLeastOneReported(true)
	}
}
Example #5
0
func (aci *Aci) Test() error {
	defer aci.giveBackUserRightsToTarget()
	hashAcis, err := aci.Install()
	if err != nil {
		return err
	}

	logs.WithF(aci.fields).Info("Testing")

	ImportInternalTesterIfNeeded(aci.manifest)

	logs.WithF(aci.fields).Info("Building test aci")
	hashTestAci, err := aci.buildTestAci()
	if err != nil {
		return err
	}

	logs.WithF(aci.fields).Info("Running test aci")
	if err := aci.runTestAci(hashTestAci, hashAcis); err != nil {
		return err
	}

	logs.WithF(aci.fields).Info("Checking result")
	if err := aci.checkResult(); err != nil {
		return err
	}
	return nil
}
Example #6
0
// this reuse zk connection if host list is the same
// a new dedicated event chan is created for each call
// zk events are duplicated to all those channels
func NewSharedZkConnection(hosts []string, timeout time.Duration) (*SharedZkConnection, error) {
	zkConnectionsMutex.Lock()
	defer zkConnectionsMutex.Unlock()

	sort.Strings(hosts)
	hash := strings.Join(hosts, "")
	if _, ok := zkConnections[hash]; !ok {
		conn, channel, err := zk.Connect(hosts, timeout)
		conn.SetLogger(ZKLogger{})
		zkConnections[hash] = &SharedZkConnection{
			hash:       hash,
			Conn:       conn,
			err:        err,
			sourceChan: channel,
		}

		go func(sharedZk *SharedZkConnection) {
			events := sharedZk.Subscribe()
			connectingCount := 0
			for {
				select {
				case e, ok := <-events:
					if !ok {
						return
					}
					if e.Type == zk.EventSession && e.State == zk.StateHasSession {
						if sharedZk.connected == false {
							logs.WithF(data.WithField("servers", hosts)).Info("Connected to zk")
							connectingCount = 0
						}
						sharedZk.connected = true
					} else if (e.Type == zk.EventSession || e.Type == zk.EventType(0)) &&
						(e.State == zk.StateDisconnected || e.State == zk.StateExpired) {
						if sharedZk.connected == true {
							logs.WithF(data.WithField("servers", hosts)).Warn("Connection lost to zk")
							connectingCount = 0
						}
						sharedZk.connected = false
					} else if (e.Type == zk.EventSession || e.Type == zk.EventType(0)) &&
						(e.State == zk.StateAuthFailed) {
						logs.WithF(data.WithField("servers", hosts)).Error("Authentication failure on zk")
					} else if (e.Type == zk.EventSession || e.Type == zk.EventType(0)) &&
						(e.State == zk.StateConnecting) {
						if connectingCount == 1 {
							logs.WithF(data.WithField("server", conn.Server())).Warn("Failed to connect to zk. Not reporting nexts servers try until connected")
						}
						connectingCount++
					}
				}
			}
		}(zkConnections[hash])
	}
	go zkConnections[hash].recipientListPublish()

	return zkConnections[hash], zkConnections[hash].err
}
Example #7
0
func (hap *HaProxyClient) SocketUpdate() error {
	if hap.socketPath == "" {
		return errs.WithF(hap.fields, "No socket file specified. Cannot update")
	}
	logs.WithF(hap.fields).Debug("Updating haproxy by socket")

	if err := hap.writeConfig(); err != nil { // just to stay in sync
		logs.WithEF(err, hap.fields).Warn("Failed to write configuration file")
	}

	conn, err := net.Dial("unix", hap.socketPath)
	if err != nil {
		return errs.WithEF(err, hap.fields.WithField("socket", hap.socketPath), "Failed to connect to haproxy socket")
	}
	defer conn.Close()

	i := 0
	b := bytes.Buffer{}
	for name, servers := range hap.Backend {
		for _, server := range servers {
			res := hap.weightRegex.FindStringSubmatch(server)
			if len(res) == 3 {
				i++
				b.WriteString("set weight " + name + "/" + res[1] + " " + res[2] + "\n")
			}
		}
	}

	if b.Len() == 0 {
		logs.WithF(hap.fields).Debug("Nothing to update by socket. No weight set")
		return nil
	}

	commands := b.Bytes()

	logs.WithF(hap.fields.WithField("command", string(commands))).Trace("Running command on hap socket")
	count, err := conn.Write(commands)
	if count != len(commands) || err != nil {
		return errs.WithEF(err, hap.fields.
			WithField("written", count).
			WithField("len", len(commands)).
			WithField("command", string(commands)), "Failed to write command to haproxy")
	}

	buff := bufio.NewReader(conn)
	line, prefix, err := buff.ReadLine()
	if err != nil || prefix {
		return errs.WithEF(err, hap.fields.WithField("line-too-long", prefix), "Failed to read hap socket response")
	}
	if string(line) != "" {
		return errs.WithF(hap.fields.WithField("response", string(line)), "Bad response for haproxy socket command")
	}

	return nil
}
Example #8
0
File: aci.go Project: puckel/dgr
func NewAciWithManifest(path string, args BuildArgs, manifestTmpl string, checkWg *sync.WaitGroup) (*Aci, error) {
	manifest, err := common.ProcessManifestTemplate(manifestTmpl, nil, false)
	if err != nil {
		return nil, errs.WithEF(err, data.WithField("content", manifestTmpl), "Failed to process manifest")
	}
	if manifest.NameAndVersion == "" {
		logs.WithField("path", path).Fatal("name is mandatory in manifest")
	}

	fields := data.WithField("aci", manifest.NameAndVersion.String())
	logs.WithF(fields).WithFields(data.Fields{"args": args, "path": path, "manifest": manifest}).Debug("New aci")

	fullPath, err := filepath.Abs(path)
	if err != nil {
		return nil, errs.WithEF(err, fields, "Cannot get fullpath of project")
	}

	target := fullPath + pathTarget
	if Home.Config.TargetWorkDir != "" {
		currentAbsDir, err := filepath.Abs(Home.Config.TargetWorkDir + "/" + manifest.NameAndVersion.ShortName())
		if err != nil {
			return nil, errs.WithEF(err, fields.WithField("path", path), "Invalid target path")
		}
		target = currentAbsDir
	}

	aci := &Aci{
		fields:          fields,
		args:            args,
		path:            fullPath,
		manifestTmpl:    manifestTmpl,
		manifest:        manifest,
		target:          target,
		FullyResolveDep: true,
		checkWg:         checkWg,
	}

	froms, err := manifest.GetFroms()
	if err != nil {
		logs.WithEF(err, aci.fields).Fatal("Invalid from data")
	}
	if len(froms) != 0 {
		if froms[0].String() == "" {
			logs.WithF(aci.fields).Warn("From is deprecated and empty, remove it")
		} else {
			logs.WithF(aci.fields).Warn("From is deprecated and processed as dependency. move from to dependencies")
			aci.manifest.Aci.Dependencies = append(froms, aci.manifest.Aci.Dependencies...)
		}
	}

	return aci, nil
}
func (w *WatcherZookeeper) watchNode(node string, stop <-chan struct{}, doneWaiter *sync.WaitGroup) {
	doneWaiter.Add(1)
	defer doneWaiter.Done()

	fields := w.fields.WithField("node", node)
	logs.WithF(fields).Debug("New node watcher")

	for {
		content, stats, childEvent, err := w.connection.Conn.GetW(node)
		if err != nil {
			if err == zk.ErrNoNode {
				logs.WithEF(err, fields).Warn("Node disappear before watching")
				w.reports.removeNode(node)
				return
			}
			w.service.synapse.watcherFailures.WithLabelValues(w.service.Name, PrometheusLabelWatch).Inc()
			logs.WithEF(err, fields).Warn("Failed to watch node, retry in 1s")
			<-time.After(time.Duration(1000) * time.Millisecond)

			if isStopped(stop) {
				return
			}
			continue
		}

		w.reports.addRawReport(node, content, fields, stats.Ctime)

		//if context.oneshot {
		//	go func() {
		//		<-rootEvents
		//	}()
		//	return
		//}

		select {
		case e := <-childEvent:
			logs.WithF(fields.WithField("event", e)).Trace("Receiving event from node")
			switch e.Type {
			case zk.EventNodeDataChanged | zk.EventNodeCreated | zk.EventNotWatching:
			// loop
			case zk.EventNodeDeleted:
				logs.WithF(fields).Debug("Node deleted")
				w.reports.removeNode(node)
				return
			}
		case <-stop:
			return
		}

	}
}
Example #10
0
func (aci *Aci) RunBuilderCommand(command common.BuilderCommand) error {
	defer aci.giveBackUserRightsToTarget()
	logs.WithF(aci.fields).Info("Building")

	if err := os.MkdirAll(aci.target, 0777); err != nil {
		return errs.WithEF(err, aci.fields, "Cannot create target directory")
	}

	if err := ioutil.WriteFile(aci.target+common.PathManifestYmlTmpl, []byte(aci.manifestTmpl), 0644); err != nil {
		return errs.WithEF(err, aci.fields.WithField("file", aci.target+common.PathManifestYmlTmpl), "Failed to write manifest template")
	}

	stage1Hash, err := aci.prepareStage1aci()
	if err != nil {
		return errs.WithEF(err, aci.fields, "Failed to prepare stage1 image")
	}

	builderHash, err := aci.prepareBuildAci()
	if err != nil {
		return errs.WithEF(err, aci.fields, "Failed to prepare build image")
	}

	logs.WithF(aci.fields).Info("Calling rkt to start build")
	defer aci.cleanupRun(builderHash, stage1Hash)
	if err := Home.Rkt.Run(aci.prepareRktRunArguments(command, builderHash, stage1Hash)); err != nil {
		return errs.WithEF(err, aci.fields, "Builder container return with failed status")
	}

	content, err := common.ExtractManifestContentFromAci(aci.target + pathImageAci)
	if err != nil {
		logs.WithEF(err, aci.fields).Warn("Failed to write manifest.json")
	}

	if err := ioutil.WriteFile(aci.target+pathManifestJson, content, 0644); err != nil {
		logs.WithEF(err, aci.fields).Warn("Failed to write manifest.json")
	}

	im := &schema.ImageManifest{}
	if err = im.UnmarshalJSON(content); err != nil {
		return errs.WithEF(err, aci.fields.WithField("content", string(content)), "Cannot unmarshall json content")
	}

	fullname := common.ExtractNameVersionFromManifest(im)
	logs.WithField("fullname", *fullname).Info("Finished building aci")
	if err := ioutil.WriteFile(aci.target+pathVersion, []byte(*fullname), 0644); err != nil {
		return errs.WithEF(err, aci.fields, "Failed to write version file in target")
	}

	return nil
}
Example #11
0
func (w *WatcherZookeeper) watchRoot(stop <-chan struct{}, doneWaiter *sync.WaitGroup) {
	doneWaiter.Add(1)
	defer doneWaiter.Done()

	for {
		childs, _, rootEvents, err := w.connection.Conn.ChildrenW(w.Path)
		if err != nil {
			w.service.synapse.watcherFailures.WithLabelValues(w.service.Name, PrometheusLabelWatch).Inc()
			logs.WithEF(err, w.fields.WithField("path", w.Path)).Warn("Cannot watch root service path. Retry in 1s")
			<-time.After(time.Duration(1000) * time.Millisecond)

			if isStopped(stop) {
				return
			}
			continue
		}

		if len(childs) == 0 {
			w.reports.setNoNodes()
		} else {
			for _, child := range childs {
				if _, ok := w.reports.get(w.Path + "/" + child); !ok {
					go w.watchNode(w.Path+"/"+child, stop, doneWaiter)
				}
			}
		}

		//if context.oneshot {
		//	go func() {
		//		<-rootEvents
		//	}()
		//	return
		//}

		select {
		case e := <-rootEvents:
			logs.WithF(w.fields.WithField("event", e)).Trace("Receiving event for root node")
			switch e.Type {
			case zk.EventNodeChildrenChanged | zk.EventNodeCreated | zk.EventNodeDataChanged | zk.EventNotWatching:
			// loop
			case zk.EventNodeDeleted:
				logs.WithF(w.fields.WithField("node", w.Path)).Debug("Rootnode deleted")
				w.reports.removeAll()
			}
		case <-stop:
			return
		}
	}
}
Example #12
0
func (aci *Aci) Push() error {
	defer aci.giveBackUserRightsToTarget()
	if Home.Config.Push.Type == "" {
		return errs.WithF(aci.fields, "Cannot push, push is not configured in dgr global configuration file")
	}

	if err := aci.EnsureBuilt(); err != nil {
		return err
	}

	if aci.args.Test {
		aci.args.Test = false
		if err := aci.Test(); err != nil {
			return err
		}
	}

	logs.WithF(aci.fields).Info("Gzipping aci before upload")

	im, err := common.ExtractManifestFromAci(aci.target + pathImageAci)
	if err != nil {
		return errs.WithEF(err, aci.fields.WithField("file", pathImageAci), "Failed to extract manifest from aci file")
	}
	val, ok := im.Labels.Get("version")
	if !ok {
		return errs.WithEF(err, aci.fields.WithField("file", pathImageAci), "Failed to get version from aci manifest")
	}

	if err := aci.zipAci(); err != nil {
		return errs.WithEF(err, aci.fields, "Failed to zip aci")
	}

	logs.WithF(aci.fields).Info("Uploading aci")
	if err := common.ExecCmd("curl", "-f", "-i",
		"-F", "r=releases",
		"-F", "hasPom=false",
		"-F", "e=aci",
		"-F", "g=com.blablacar.aci.linux.amd64",
		"-F", "p=aci",
		"-F", "v="+val,
		"-F", "a="+strings.Split(string(im.Name), "/")[1],
		"-F", "file=@"+aci.target+pathImageGzAci,
		"-u", Home.Config.Push.Username+":"+Home.Config.Push.Password,
		Home.Config.Push.Url+"/service/local/artifact/maven/content"); err != nil {
		return errs.WithEF(err, aci.fields, "Failed to push aci")
	}
	return nil
}
Example #13
0
func (cnt *Aci) checkResult() {
	files, err := ioutil.ReadDir(cnt.target + PATH_TESTS + PATH_TARGET + PATH_RESULT)
	if err != nil {
		panic("Cannot read test result directory" + err.Error())
	}
	testFound := false
	for _, f := range files {
		fullPath := cnt.target + PATH_TESTS + PATH_TARGET + PATH_RESULT + "/" + f.Name()
		content, err := ioutil.ReadFile(fullPath)
		if err != nil {
			panic("Cannot read result file" + f.Name() + err.Error())
		}
		if !strings.HasSuffix(f.Name(), STATUS_SUFFIX) {
			if testFound == false && string(content) != "1..0\n" {
				testFound = true
			}
			continue
		}
		if string(content) != "0\n" {
			logs.WithF(cnt.fields).WithField("file", f.Name()).Error("Failed test")
			os.Exit(2)
		}
	}

	if cnt.args.NoTestFail && !testFound {
		panic("No tests found")
	}
}
Example #14
0
func Logger() macaron.Handler {
	var reqCounter int64
	return func(ctx *macaron.Context, log *log.Logger) {
		start := time.Now()

		fields := data.WithField("method", ctx.Req.Method).
			WithField("uri", ctx.Req.RequestURI).
			WithField("ip", ctx.RemoteAddr()).
			WithField("id", atomic.AddInt64(&reqCounter, 1))
		if logs.IsDebugEnabled() {
			logs.WithF(fields).Trace("Request received")
		}

		rw := ctx.Resp.(macaron.ResponseWriter)
		ctx.Next()

		if logs.IsInfoEnabled() {
			fields = fields.WithField("duration", time.Since(start)).WithField("status", rw.Status())
			var lvl logs.Level
			if rw.Status() >= 500 && rw.Status() < 600 {
				lvl = logs.ERROR
			} else {
				lvl = logs.DEBUG
			}

			logs.LogEntry(&logs.Entry{
				Fields:  fields,
				Level:   lvl,
				Message: "Request completed",
			})

		}
	}
}
Example #15
0
func (c *CheckCommon) CommonRun(checker Checker, statusChange chan<- Check, stop <-chan struct{}, doneWait *sync.WaitGroup) {
	logs.WithF(c.fields).Info("Starting check")
	doneWait.Add(1)
	defer doneWait.Done()

	for {
		status := checker.Check()
		if logs.IsTraceEnabled() {
			logs.WithEF(status, c.fields).Trace("Check done")
		}
		if status != nil {
			logs.WithEF(status, c.fields).Debug("Failed check")
		}
		if status != nil && !c.service.NoMetrics {
			c.service.nerve.checkerFailureCount.WithLabelValues(c.service.Name, c.Host, strconv.Itoa(c.Port), c.Type).Inc()
		}
		c.saveStatus(status)

		current := c.stableStatus
		latest := c.latestStatuses
		if (latest[0] == nil && sameLastStatusCount(latest) >= c.Rise && (current == nil || *current != nil)) ||
			(latest[0] != nil && sameLastStatusCount(latest) >= c.Fall && (current == nil || *current == nil)) {
			c.stableStatus = &status
			statusChange <- Check{checker, *c.stableStatus}
		}

		select {
		case <-stop:
			logs.WithFields(c.fields).Debug("Stopping check")
			return
		case <-time.After(time.Duration(c.CheckIntervalInMilli) * time.Millisecond):
		}
	}
}
Example #16
0
func (b *Builder) Build() error {
	logs.WithF(b.fields).Info("Building aci")

	lfd, err := rktcommon.GetRktLockFD()
	if err != nil {
		return errs.WithEF(err, b.fields, "can't get rkt lock fd")
	}

	if err := sys.CloseOnExec(lfd, true); err != nil {
		return errs.WithEF(err, b.fields, "can't set FD_CLOEXEC on rkt lock")
	}

	if err := b.runBuild(); err != nil {
		return err
	}

	if err := b.writeManifest(); err != nil {
		return err
	}

	if err := b.tarAci(); err != nil {
		return err
	}

	return nil
}
Example #17
0
func (aci *Aci) prepareBuildAci() (string, error) {
	logs.WithFields(aci.fields).Debug("Preparing builder")

	if err := os.MkdirAll(aci.target+pathBuilder+common.PathRootfs, 0777); err != nil {
		return "", errs.WithEF(err, aci.fields.WithField("path", aci.target+pathBuilder), "Failed to create builder aci path")
	}

	if err := ioutil.WriteFile(aci.target+pathBuilder+common.PathRootfs+"/.keep", []byte(""), 0644); err != nil {
		return "", errs.WithEF(err, aci.fields.WithField("file", aci.target+pathBuilder+common.PathRootfs+"/.keep"), "Failed to write keep file")
	}
	capa, err := types.NewLinuxCapabilitiesRetainSet("all")
	if err != nil {
		return "", errs.WithEF(err, aci.fields, "Failed to create all capability retain Set")
	}
	allIsolator, err := capa.AsIsolator()
	if err != nil {
		return "", errs.WithEF(err, aci.fields, "Failed to prepare all retain set isolator")
	}

	aci.manifest.Aci.App.Isolators = types.Isolators([]types.Isolator{*allIsolator})

	if err := common.WriteAciManifest(aci.manifest, aci.target+pathBuilder+common.PathManifest, common.PrefixBuilder+aci.manifest.NameAndVersion.Name(), BuildVersion); err != nil {
		return "", err
	}
	if err := aci.tarAci(aci.target + pathBuilder); err != nil {
		return "", err
	}

	logs.WithF(aci.fields.WithField("path", aci.target+pathBuilder+pathImageAci)).Info("Importing build to rkt")
	hash, err := Home.Rkt.FetchInsecure(aci.target + pathBuilder + pathImageAci)
	if err != nil {
		return "", errs.WithEF(err, aci.fields, "fetch of builder aci failed")
	}
	return hash, nil
}
Example #18
0
File: env.go Project: blablacar/ggn
func (e Env) runFleetCmdInternal(getOutput bool, args []string) (string, string, error) {
	logs.WithF(e.fields).WithField("command", strings.Join(args, " ")).Debug("Running command on fleet")
	if e.config.Fleet.Endpoint == "" {
		return "", "", errors.New("Cannot find fleet.endpoint env config to call fleetctl")
	}

	envs := map[string]string{}
	envs[FLEETCTL_ENDPOINT] = e.config.Fleet.Endpoint
	if e.config.Fleet.Username != "" {
		envs[FLEETCTL_SSH_USERNAME] = e.config.Fleet.Username
	}
	envs[FLEETCTL_STRICT_HOST_KEY_CHECKING] = fmt.Sprintf("%t", e.config.Fleet.Strict_host_key_checking)
	envs[FLEETCTL_SUDO] = fmt.Sprintf("%t", e.config.Fleet.Sudo)

	args = append([]string{"fleetctl"}, args...)
	for key, val := range envs {
		args = append([]string{key + "='" + val + "'"}, args...)
	}

	var stdout string
	var stderr string
	var err error
	if getOutput {
		stdout, stderr, err = common.ExecCmdGetStdoutAndStderr("bash", "-c", strings.Join(args, " "))
	} else {
		err = common.ExecCmd("bash", "-c", strings.Join(args, " "))
	}
	return stdout, stderr, err
}
Example #19
0
func (b *Builder) runBuild() error {
	command, err := b.getCommandPath()
	if err != nil {
		return err
	}

	logs.WithF(b.fields).Debug("Running build command")
	args, env, err := b.prepareNspawnArgsAndEnv(command)
	if err != nil {
		return err
	}

	os.Remove(b.stage1Rootfs + "/etc/machine-id")

	if logs.IsDebugEnabled() {
		logs.WithField("command", strings.Join([]string{args[0], " ", strings.Join(args[1:], " ")}, " ")).Debug("Running external command")
	}
	//	var stderr bytes.Buffer
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Env = env
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr

	if err := cmd.Run(); err != nil {
		return errs.WithEF(err, b.fields, "Builder run failed")
	}

	return nil
}
Example #20
0
func (aci *Aci) Clean() {
	logs.WithF(aci.fields).Debug("Cleaning")

	if err := os.RemoveAll(aci.target + "/"); err != nil {
		logs.WithEF(err, aci.fields).WithField("dir", aci.target).Warn("Cannot remove directory")
	}
}
Example #21
0
func (p *Pod) Push() {
	logs.WithF(p.fields).Info("Pushing")

	p.Build()

	checkVersion := make(chan bool, 1)

	for _, e := range p.manifest.Pod.Apps {
		aci, err := NewAciWithManifest(p.path+"/"+e.Name, p.args, p.toAciManifest(e), &checkVersion)
		if err != nil {
			logs.WithEF(err, p.fields.WithField("name", e.Name)).Fatal("Cannot prepare aci")
		}
		aci.podName = &p.manifest.Name
		aci.Push()
	}

	for range p.manifest.Pod.Apps {
		<-checkVersion
	}

	if err := utils.ExecCmd("curl", "-i",
		"-F", "r=releases",
		"-F", "hasPom=false",
		"-F", "e=pod",
		"-F", "g=com.blablacar.aci.linux.amd64",
		"-F", "p=pod",
		"-F", "v="+p.manifest.Name.Version(),
		"-F", "a="+p.manifest.Name.ShortName(),
		"-F", "file=@"+p.target+"/pod-manifest.json",
		"-u", cnt.Home.Config.Push.Username+":"+cnt.Home.Config.Push.Password,
		cnt.Home.Config.Push.Url+"/service/local/artifact/maven/content"); err != nil {
		logs.WithEF(err, p.fields).Fatal("Cannot push pod")
	}

}
Example #22
0
func (aci *Aci) prepareBuildAci() (string, error) {
	logs.WithFields(aci.fields).Debug("Preparing builder")

	if err := os.MkdirAll(aci.target+pathBuilder+common.PathRootfs, 0777); err != nil {
		return "", errs.WithEF(err, aci.fields.WithField("path", aci.target+pathBuilder), "Failed to create builder aci path")
	}

	if err := ioutil.WriteFile(aci.target+pathBuilder+common.PathRootfs+"/.keep", []byte(""), 0644); err != nil {
		return "", errs.WithEF(err, aci.fields.WithField("file", aci.target+pathBuilder+common.PathRootfs+"/.keep"), "Failed to write keep file")
	}

	if err := common.WriteAciManifest(aci.manifest, aci.target+pathBuilder+common.PathManifest, common.PrefixBuilder+aci.manifest.NameAndVersion.Name(), dgrVersion); err != nil {
		return "", err
	}
	if err := aci.tarAci(aci.target + pathBuilder); err != nil {
		return "", err
	}

	logs.WithF(aci.fields.WithField("path", aci.target+pathBuilder+pathImageAci)).Info("Importing build to rkt")
	hash, err := Home.Rkt.Fetch(aci.target + pathBuilder + pathImageAci)
	if err != nil {
		return "", errs.WithEF(err, aci.fields, "fetch of builder aci failed")
	}
	return hash, nil
}
Example #23
0
func (aci *Aci) copyInternals() {
	logs.WithF(aci.fields).Debug("Copy internals")
	os.MkdirAll(aci.rootfs+PATH_CNT+PATH_BIN, 0755)
	os.MkdirAll(aci.rootfs+"/bin", 0755)     // this is required or systemd-nspawn will create symlink on it
	os.MkdirAll(aci.rootfs+"/usr/bin", 0755) // this is required by systemd-nspawn

	busybox, _ := dist.Asset("dist/bindata/busybox")
	if err := ioutil.WriteFile(aci.rootfs+PATH_CNT+PATH_BIN+"/busybox", busybox, 0777); err != nil {
		panic(err)
	}

	confd, _ := dist.Asset("dist/bindata/confd")
	if err := ioutil.WriteFile(aci.rootfs+PATH_CNT+PATH_BIN+"/confd", confd, 0777); err != nil {
		panic(err)
	}

	attributeMerger, _ := dist.Asset("dist/bindata/attributes-merger")
	if err := ioutil.WriteFile(aci.rootfs+PATH_CNT+PATH_BIN+"/attributes-merger", attributeMerger, 0777); err != nil {
		panic(err)
	}

	confdFile := `backend = "env"
confdir = "/cnt"
prefix = "/confd"
log-level = "debug"
`
	os.MkdirAll(aci.rootfs+PATH_CNT+"/prestart", 0755)
	if err := ioutil.WriteFile(aci.rootfs+PATH_CNT+"/prestart/confd.toml", []byte(confdFile), 0777); err != nil {
		panic(err)
	}

	if err := ioutil.WriteFile(aci.rootfs+PATH_CNT+PATH_BIN+"/prestart", []byte(PRESTART), 0777); err != nil {
		panic(err)
	}
}
Example #24
0
func (aci *Aci) upload(name *common.ACFullname) error {
	if Home.Config.Push.Type == "maven" && name.DomainName() == "aci.blbl.cr" { // TODO this definitely need to be removed
		logs.WithF(aci.fields).Info("Uploading aci")
		if err := common.ExecCmd("curl", "-f", "-i",
			"-F", "r=releases",
			"-F", "hasPom=false",
			"-F", "e=aci",
			"-F", "g=com.blablacar.aci.linux.amd64",
			"-F", "p=aci",
			"-F", "v="+name.Version(),
			"-F", "a="+strings.Split(string(name.Name()), "/")[1],
			"-F", "file=@"+aci.target+pathImageGzAci,
			"-u", Home.Config.Push.Username+":"+Home.Config.Push.Password,
			Home.Config.Push.Url+"/service/local/artifact/maven/content"); err != nil {
			return errs.WithEF(err, aci.fields, "Failed to push aci")
		}
	} else {
		systemConf := Home.Config.Rkt.SystemConfig
		if systemConf == "" {
			systemConf = "/usr/lib/rkt"
		}
		localConf := Home.Config.Rkt.LocalConfig
		if localConf == "" {
			localConf = "/etc/rkt"
		}

		conf, err := config.GetConfigFrom(systemConf, localConf)
		if err != nil {
			return errs.WithEF(err, aci.fields, "Failed to get rkt configuration")
		}

		upload := Uploader{
			Acipath: aci.target + pathImageGzAci,
			Ascpath: aci.target + pathImageGzAciAsc,
			Uri:     aci.manifest.NameAndVersion.String(),
			Debug:   false,
			SetHTTPHeaders: func(r *http.Request) {
				if r.URL == nil {
					return
				}
				headerer, ok := conf.AuthPerHost[r.URL.Host]
				if !ok {
					logs.WithFields(aci.fields).WithField("domain", r.URL.Host).
						Warn("No auth credential found in rkt configuration for this domain")
					return
				}
				header := headerer.Header()
				for k, v := range header {
					r.Header[k] = append(r.Header[k], v...)
				}
			},
		}
		err = upload.Upload()
		if err != nil {
			return errs.WithEF(err, aci.fields, "Failed to upload aci")
		}

	}
	return nil
}
Example #25
0
func NewAciWithManifest(path string, args BuildArgs, manifest spec.AciManifest, checked *chan bool) (*Aci, error) {
	if manifest.NameAndVersion == "" {
		logs.WithField("path", path).Fatal("name is mandatory in manifest")
	}

	fields := data.WithField("aci", manifest.NameAndVersion.String())
	logs.WithF(fields).WithFields(data.Fields{"args": args, "path": path, "manifest": manifest}).Debug("New aci")

	fullPath, err := filepath.Abs(path)
	if err != nil {
		return nil, errs.WithEF(err, fields, "Cannot get fullpath of project")
	}

	target := fullPath + PATH_TARGET
	if cnt.Home.Config.TargetWorkDir != "" {
		currentAbsDir, err := filepath.Abs(cnt.Home.Config.TargetWorkDir + "/" + manifest.NameAndVersion.ShortName())
		if err != nil {
			return nil, errs.WithEF(err, fields.WithField("path", path), "Invalid target path")
		}
		target = currentAbsDir
	}

	aci := &Aci{
		fields:          fields,
		args:            args,
		path:            fullPath,
		manifest:        manifest,
		target:          target,
		rootfs:          target + PATH_ROOTFS,
		FullyResolveDep: true,
	}

	go aci.checkLatestVersions(checked)
	return aci, nil
}
Example #26
0
func (aci *Aci) prepareBuildAci() (string, error) {
	logs.WithFields(aci.fields).Debug("Preparing builder")

	if err := os.MkdirAll(aci.target+pathBuilder+common.PathRootfs, 0777); err != nil {
		return "", errs.WithEF(err, aci.fields.WithField("path", aci.target+pathBuilder), "Failed to create builder aci path")
	}

	if err := ioutil.WriteFile(aci.target+pathBuilder+common.PathRootfs+"/.keep", []byte(""), 0644); err != nil {
		return "", errs.WithEF(err, aci.fields.WithField("file", aci.target+pathBuilder+common.PathRootfs+"/.keep"), "Failed to write keep file")
	}
	aci.manifest.Aci.App.Isolators = []common.Isolator{{Name: "os/linux/capabilities-retain-set", Value: common.LinuxCapabilitiesSetValue{Set: []types.LinuxCapability{"all"}}}}

	if err := common.WriteAciManifest(aci.manifest, aci.target+pathBuilder+common.PathManifest, common.PrefixBuilder+aci.manifest.NameAndVersion.Name(), dgrVersion); err != nil {
		return "", err
	}
	if err := aci.tarAci(aci.target + pathBuilder); err != nil {
		return "", err
	}

	logs.WithF(aci.fields.WithField("path", aci.target+pathBuilder+pathImageAci)).Info("Importing build to rkt")
	hash, err := Home.Rkt.FetchInsecure(aci.target + pathBuilder + pathImageAci)
	if err != nil {
		return "", errs.WithEF(err, aci.fields, "fetch of builder aci failed")
	}
	return hash, nil
}
Example #27
0
func ExecCommandFull(cmd []string, env []string, timeoutInMilli int) error {
	command := exec.Command(cmd[0], cmd[1:]...)
	var b bytes.Buffer
	command.Stdout = &b
	command.Stderr = &b
	command.Env = env

	if err := command.Start(); err != nil {
		return errs.WithEF(err, data.WithField("cmd", cmd), "Failed to start command")
	}

	var after *errs.EntryError
	timer := time.AfterFunc(time.Duration(timeoutInMilli)*time.Millisecond, func() {
		data := data.WithField("command", strings.Join(cmd, " ")).WithField("timeout", timeoutInMilli)
		logs.WithF(data).Debug("Command timeout")
		after = errs.WithF(data, "Exec command timeout")
		command.Process.Kill()
	})

	err := command.Wait()
	timer.Stop()
	if logs.IsTraceEnabled() {
		logs.WithField("cmd", cmd).WithField("output", string(b.Bytes())).Trace("Command output")
	}
	if err != nil {
		return errs.WithEF(err, data.WithField("cmd", cmd).
			WithField("output", string(b.Bytes())), "Command failed").
			WithErr(after)
	}
	return nil
}
Example #28
0
func (s *Service) Start(stopper <-chan struct{}, stopWait *sync.WaitGroup) {
	logs.WithFields(s.fields).Info("Starting service check")
	stopWait.Add(1)
	defer stopWait.Done()
	checkStopWait := &sync.WaitGroup{}

	statusChange := make(chan Check, 2)
	for checker := range s.typedCheckersWithStatus {
		go checker.Run(statusChange, stopper, checkStopWait)
	}

	for {
		select {
		case status := <-statusChange:
			logs.WithF(s.fields.WithField("status", status)).Debug("New status received")
			s.processCheckResult(status)
		case <-stopper: //TODO since stop is the same everywhere, statusChange chan may stay stuck on shutdown
			logs.WithFields(s.fields).Debug("Stop requested")
			checkStopWait.Wait()
			close(statusChange)
			if *s.SetServiceAsDownOnShutdown {
				wait := &sync.WaitGroup{}
				wait.Add(1)
				s.Disable(wait, false)
				wait.Wait()
			}
			for reporter := range s.typedReportersWithReported {
				reporter.Destroy()
			}
			return
		case <-time.After(time.Duration(s.ReportReplayInMilli) * time.Millisecond):
			s.reportAndTellIfAtLeastOneReported(false)
		}
	}
}
Example #29
0
func (p *Pod) Graph() {
	logs.WithF(p.fields).Info("Graphing")
	os.MkdirAll(p.target, 0777)

	var buffer bytes.Buffer
	buffer.WriteString("digraph {\n")
	buffer.WriteString("  {\n")
	buffer.WriteString("  ")
	buffer.WriteString("\"")
	buffer.WriteString(p.manifest.Name.ShortNameId())
	buffer.WriteString("\"")
	buffer.WriteString(" [style=filled, fillcolor=yellow, shape=box]\n")
	buffer.WriteString("  }\n")

	for _, e := range p.manifest.Pod.Apps {
		for _, d := range e.Dependencies {
			buffer.WriteString("  ")
			buffer.WriteString("\"")
			buffer.WriteString(d.ShortNameId())
			buffer.WriteString("\"")
			buffer.WriteString(" -> ")
			buffer.WriteString("\"")
			buffer.WriteString(p.manifest.Name.ShortNameId())
			buffer.WriteString("\"")
			buffer.WriteString("\n")
		}
	}

	buffer.WriteString("}\n")

	ioutil.WriteFile(p.target+"/graph.dot", buffer.Bytes(), 0644)

}
Example #30
0
func (p *Pod) Install() ([]string, error) {
	logs.WithF(p.fields).Info("Installing")

	hashs := []string{}

	if err := p.CleanAndBuild(); err != nil {
		return hashs, err
	}

	for _, e := range p.manifest.Pod.Apps {
		tmpl, err := p.toAciManifestTemplate(e)
		if err != nil {
			return nil, err
		}

		aci, err := NewAciWithManifest(p.path+"/"+e.Name, p.args, tmpl, p.checkWg)
		if err != nil {
			logs.WithEF(err, p.fields.WithField("name", e.Name)).Fatal("Cannot prepare aci")
		}
		aci.podName = &p.manifest.Name
		hash, err := aci.Install()
		if err != nil {
			return hashs, err
		}
		hashs = append(hashs, hash...)
	}
	return hashs, nil
}