func ExecCommandFull(cmd []string, env []string, timeoutInMilli int) error {
command := exec.Command(cmd[0], cmd[1:]...)
var b bytes.Buffer
command.Stdout = &b
command.Stderr = &b
command.Env = env
if err := command.Start(); err != nil {
return errs.WithEF(err, data.WithField("cmd", cmd), "Failed to start command")
}
var after *errs.EntryError
timer := time.AfterFunc(time.Duration(timeoutInMilli)*time.Millisecond, func() {
data := data.WithField("command", strings.Join(cmd, " ")).WithField("timeout", timeoutInMilli)
logs.WithF(data).Debug("Command timeout")
after = errs.WithF(data, "Exec command timeout")
command.Process.Kill()
})
err := command.Wait()
timer.Stop()
if logs.IsTraceEnabled() {
logs.WithField("cmd", cmd).WithField("output", string(b.Bytes())).Trace("Command output")
}
if err != nil {
return errs.WithEF(err, data.WithField("cmd", cmd).
WithField("output", string(b.Bytes())), "Command failed").
WithErr(after)
}
return nil
}
func NewHome(path string) HomeStruct {
logs.WithField("path", path).Debug("Loading home")
var config Config
if source, err := ioutil.ReadFile(path + "/config.yml"); err == nil {
err = yaml.Unmarshal([]byte(source), &config)
if err != nil {
logs.WithEF(err, data.WithField("path", path+"/config.yml")).Fatal("Failed to process configuration file")
}
} else if source, err := ioutil.ReadFile(DefaultHomeFolder("cnt") + "/config.yml"); err == nil {
logs.WithField("old", DefaultHomeFolder("cnt")+"/config.yml").WithField("new", DefaultHomeFolder("")).Warn("You are using old home folder")
err = yaml.Unmarshal([]byte(source), &config)
if err != nil {
logs.WithEF(err, data.WithField("path", path+"/config.yml")).Fatal("Failed to process configuration file")
}
}
if Args.NoStore {
config.Rkt.NoStore = true
}
if Args.StoreOnly {
config.Rkt.StoreOnly = true
}
rkt, err := common.NewRktClient(config.Rkt)
if err != nil {
logs.WithEF(err, data.WithField("config", config.Rkt)).Fatal("Rkt access failed")
}
return HomeStruct{
path: path,
Config: config,
Rkt: rkt,
}
}
// this reuse zk connection if host list is the same
// a new dedicated event chan is created for each call
// zk events are duplicated to all those channels
func NewSharedZkConnection(hosts []string, timeout time.Duration) (*SharedZkConnection, error) {
zkConnectionsMutex.Lock()
defer zkConnectionsMutex.Unlock()
sort.Strings(hosts)
hash := strings.Join(hosts, "")
if _, ok := zkConnections[hash]; !ok {
conn, channel, err := zk.Connect(hosts, timeout)
conn.SetLogger(ZKLogger{})
zkConnections[hash] = &SharedZkConnection{
hash: hash,
Conn: conn,
err: err,
sourceChan: channel,
}
go func(sharedZk *SharedZkConnection) {
events := sharedZk.Subscribe()
connectingCount := 0
for {
select {
case e, ok := <-events:
if !ok {
return
}
if e.Type == zk.EventSession && e.State == zk.StateHasSession {
if sharedZk.connected == false {
logs.WithF(data.WithField("servers", hosts)).Info("Connected to zk")
connectingCount = 0
}
sharedZk.connected = true
} else if (e.Type == zk.EventSession || e.Type == zk.EventType(0)) &&
(e.State == zk.StateDisconnected || e.State == zk.StateExpired) {
if sharedZk.connected == true {
logs.WithF(data.WithField("servers", hosts)).Warn("Connection lost to zk")
connectingCount = 0
}
sharedZk.connected = false
} else if (e.Type == zk.EventSession || e.Type == zk.EventType(0)) &&
(e.State == zk.StateAuthFailed) {
logs.WithF(data.WithField("servers", hosts)).Error("Authentication failure on zk")
} else if (e.Type == zk.EventSession || e.Type == zk.EventType(0)) &&
(e.State == zk.StateConnecting) {
if connectingCount == 1 {
logs.WithF(data.WithField("server", conn.Server())).Warn("Failed to connect to zk. Not reporting nexts servers try until connected")
}
connectingCount++
}
}
}
}(zkConnections[hash])
}
go zkConnections[hash].recipientListPublish()
return zkConnections[hash], zkConnections[hash].err
}
func WritePodManifest(im *schema.PodManifest, targetFile string) error {
buff, err := json.MarshalIndent(im, "", " ")
if err != nil {
return errs.WithEF(err, data.WithField("object", im), "Failed to marshal manifest")
}
err = ioutil.WriteFile(targetFile, []byte(buff), 0644)
if err != nil {
return errs.WithEF(err, data.WithField("file", targetFile), "Failed to write pod manifest")
}
return nil
}
func NewAciWithManifest(path string, args BuildArgs, manifestTmpl string, checkWg *sync.WaitGroup) (*Aci, error) {
manifest, err := common.ProcessManifestTemplate(manifestTmpl, nil, false)
if err != nil {
return nil, errs.WithEF(err, data.WithField("content", manifestTmpl), "Failed to process manifest")
}
if manifest.NameAndVersion == "" {
logs.WithField("path", path).Fatal("name is mandatory in manifest")
}
fields := data.WithField("aci", manifest.NameAndVersion.String())
logs.WithF(fields).WithFields(data.Fields{"args": args, "path": path, "manifest": manifest}).Debug("New aci")
fullPath, err := filepath.Abs(path)
if err != nil {
return nil, errs.WithEF(err, fields, "Cannot get fullpath of project")
}
target := fullPath + pathTarget
if Home.Config.TargetWorkDir != "" {
currentAbsDir, err := filepath.Abs(Home.Config.TargetWorkDir + "/" + manifest.NameAndVersion.ShortName())
if err != nil {
return nil, errs.WithEF(err, fields.WithField("path", path), "Invalid target path")
}
target = currentAbsDir
}
aci := &Aci{
fields: fields,
args: args,
path: fullPath,
manifestTmpl: manifestTmpl,
manifest: manifest,
target: target,
FullyResolveDep: true,
checkWg: checkWg,
}
froms, err := manifest.GetFroms()
if err != nil {
logs.WithEF(err, aci.fields).Fatal("Invalid from data")
}
if len(froms) != 0 {
if froms[0].String() == "" {
logs.WithF(aci.fields).Warn("From is deprecated and empty, remove it")
} else {
logs.WithF(aci.fields).Warn("From is deprecated and processed as dependency. move from to dependencies")
aci.manifest.Aci.Dependencies = append(froms, aci.manifest.Aci.Dependencies...)
}
}
return aci, nil
}
func LoadConfig(configPath string) (*synapse.Synapse, error) {
file, err := ioutil.ReadFile(configPath)
if err != nil {
return nil, errs.WithEF(err, data.WithField("file", configPath), "Failed to read configuration file")
}
conf := &synapse.Synapse{}
err = yaml.Unmarshal(file, conf)
if err != nil {
return nil, errs.WithEF(err, data.WithField("file", configPath), "Invalid configuration format")
}
return conf, nil
}
func templateAttribute(text string, attributes interface{}) (string, error) {
tmpl, err := template.New("").Funcs(tpl.TemplateFunctions).Funcs(map[string]interface{}(gtf.GtfFuncMap)).Parse(text)
if err != nil {
return "", errs.WithEF(err, data.WithField("attribute", text), "Failed to parse template for attribute")
}
var b bytes.Buffer
if err := tmpl.Execute(&b, attributes); err != nil {
return "", errs.WithEF(err, data.WithField("attribute", text), "Failed to template attribute")
}
res := b.String()
if logs.IsDebugEnabled() {
logs.WithField("from", text).WithField("to", res).Debug("attribute templated")
}
return res, nil
}
func NewAciWithManifest(path string, args BuildArgs, manifest spec.AciManifest, checked *chan bool) (*Aci, error) {
if manifest.NameAndVersion == "" {
logs.WithField("path", path).Fatal("name is mandatory in manifest")
}
fields := data.WithField("aci", manifest.NameAndVersion.String())
logs.WithF(fields).WithFields(data.Fields{"args": args, "path": path, "manifest": manifest}).Debug("New aci")
fullPath, err := filepath.Abs(path)
if err != nil {
return nil, errs.WithEF(err, fields, "Cannot get fullpath of project")
}
target := fullPath + PATH_TARGET
if cnt.Home.Config.TargetWorkDir != "" {
currentAbsDir, err := filepath.Abs(cnt.Home.Config.TargetWorkDir + "/" + manifest.NameAndVersion.ShortName())
if err != nil {
return nil, errs.WithEF(err, fields.WithField("path", path), "Invalid target path")
}
target = currentAbsDir
}
aci := &Aci{
fields: fields,
args: args,
path: fullPath,
manifest: manifest,
target: target,
rootfs: target + PATH_ROOTFS,
FullyResolveDep: true,
}
go aci.checkLatestVersions(checked)
return aci, nil
}
func ExtractManifestContentFromAci(aciPath string) ([]byte, error) {
fields := data.WithField("file", aciPath)
input, err := os.Open(aciPath)
if err != nil {
return nil, errs.WithEF(err, fields, "Cannot open file")
}
defer input.Close()
tr, err := aci.NewCompressedTarReader(input)
if err != nil {
return nil, errs.WithEF(err, fields, "Cannot open file as tar")
}
Tar:
for {
hdr, err := tr.Next()
switch err {
case io.EOF:
break Tar
case nil:
if filepath.Clean(hdr.Name) == aci.ManifestFile {
bytes, err := ioutil.ReadAll(tr)
if err != nil {
return nil, errs.WithEF(err, fields, "Cannot read manifest content in tar")
}
return bytes, nil
}
default:
return nil, errs.WithEF(err, fields, "error reading tarball file")
}
}
return nil, errs.WithEF(err, fields, "Cannot found manifest in file")
}
func RouterFromJson(content []byte, s *Synapse) (Router, error) {
t := &RouterCommon{}
if err := json.Unmarshal([]byte(content), t); err != nil {
return nil, errs.WithE(err, "Failed to unmarshall check type")
}
fields := data.WithField("type", t.Type)
var typedRouter Router
switch t.Type {
case "console":
typedRouter = NewRouterConsole()
case "haproxy":
typedRouter = NewRouterHaProxy()
case "template":
typedRouter = NewRouterTemplate()
default:
return nil, errs.WithF(fields, "Unsupported router type")
}
if err := json.Unmarshal([]byte(content), &typedRouter); err != nil {
return nil, errs.WithEF(err, fields, "Failed to unmarshall router")
}
if err := typedRouter.Init(s); err != nil {
return nil, errs.WithEF(err, fields, "Failed to init router")
}
return typedRouter, nil
}
func NewBuilder(podRoot string, podUUID *types.UUID) (*Builder, error) {
pod, err := stage1commontypes.LoadPod(podRoot, podUUID)
if err != nil {
logs.WithError(err).Fatal("Failed to load pod")
}
if len(pod.Manifest.Apps) != 1 {
logs.Fatal("dgr builder support only 1 application")
}
fields := data.WithField("aci", manifestApp(pod).Name)
aciPath, ok := manifestApp(pod).App.Environment.Get(common.EnvAciPath)
if !ok || aciPath == "" {
return nil, errs.WithF(fields, "Builder image require "+common.EnvAciPath+" environment variable")
}
aciTarget, ok := manifestApp(pod).App.Environment.Get(common.EnvAciTarget)
if !ok || aciPath == "" {
return nil, errs.WithF(fields, "Builder image require "+common.EnvAciTarget+" environment variable")
}
return &Builder{
fields: fields,
aciHomePath: aciPath,
aciTargetPath: aciTarget,
pod: pod,
stage1Rootfs: rktcommon.Stage1RootfsPath(pod.Root),
stage2Rootfs: filepath.Join(rktcommon.AppPath(pod.Root, manifestApp(pod).Name), "rootfs"),
}, nil
}
func NewAci(path string, args BuildArgs) (*Aci, error) {
manifest, err := readAciManifest(path + PATH_CNT_MANIFEST)
if err != nil {
return nil, errs.WithEF(err, data.WithField("path", path+PATH_CNT_MANIFEST), "Cannot read manifest")
}
return NewAciWithManifest(path, args, *manifest, nil)
}
func NewAci(path string, args BuildArgs, checkWg *sync.WaitGroup) (*Aci, error) {
manifest, err := ioutil.ReadFile(path + common.PathAciManifest)
if err != nil {
return nil, errs.WithEF(err, data.WithField("path", path+common.PathAciManifest), "Cannot read manifest")
}
return NewAciWithManifest(path, args, string(manifest), checkWg)
}
func Logger() macaron.Handler {
var reqCounter int64
return func(ctx *macaron.Context, log *log.Logger) {
start := time.Now()
fields := data.WithField("method", ctx.Req.Method).
WithField("uri", ctx.Req.RequestURI).
WithField("ip", ctx.RemoteAddr()).
WithField("id", atomic.AddInt64(&reqCounter, 1))
if logs.IsDebugEnabled() {
logs.WithF(fields).Trace("Request received")
}
rw := ctx.Resp.(macaron.ResponseWriter)
ctx.Next()
if logs.IsInfoEnabled() {
fields = fields.WithField("duration", time.Since(start)).WithField("status", rw.Status())
var lvl logs.Level
if rw.Status() >= 500 && rw.Status() < 600 {
lvl = logs.ERROR
} else {
lvl = logs.DEBUG
}
logs.LogEntry(&logs.Entry{
Fields: fields,
Level: lvl,
Message: "Request completed",
})
}
}
}
func (n ACFullname) LatestVersion() (string, error) {
app, err := discovery.NewAppFromString(n.Name() + ":latest")
if app.Labels["os"] == "" {
app.Labels["os"] = "linux"
}
if app.Labels["arch"] == "" {
app.Labels["arch"] = "amd64"
}
endpoints, _, err := discovery.DiscoverACIEndpoints(*app, nil, discovery.InsecureTLS|discovery.InsecureHTTP) //TODO support security
if err != nil {
return "", errors.Annotate(err, "Latest discovery fail")
}
r, _ := regexp.Compile(`^\d+(.\d+){0,2}(-[\.\-\dA-Za-z]+){0,1}$`) // TODO this is nexus specific
if len(endpoints) == 0 {
return "", errs.WithF(data.WithField("aci", string(n)), "Discovery does not give an endpoint to check latest version")
}
url := getRedirectForLatest(endpoints[0].ACI)
logs.WithField("url", url).Debug("latest verion url")
for _, part := range strings.Split(url, "/") {
if r.Match([]byte(part)) {
return part, nil
}
}
return "", errors.New("No latest version found")
}
func (c *CheckCommon) CommonInit(s *Service) error {
c.service = s
if c.TimeoutInMilli == 0 {
c.TimeoutInMilli = 1000
}
if c.Rise == 0 {
c.Rise = 3
}
if c.Fall == 0 {
c.Fall = 2
}
if c.CheckIntervalInMilli == 0 {
c.CheckIntervalInMilli = 1000
}
if c.Port == 0 {
c.Port = s.Port
}
if c.Host == "" {
c.Host = s.Host
}
if c.Host == "" {
c.Host = "127.0.0.1"
} else if net.ParseIP(s.Host) == nil {
c.Host = IpLookupNoError(c.Host, s.PreferIpv4).String()
}
c.fields = data.WithField("type", c.Type).WithFields(s.fields)
return nil
}
func NewPod(path string, args BuildArgs) (*Pod, error) {
fullPath, err := filepath.Abs(path)
if err != nil {
logs.WithE(err).WithField("path", path).Fatal("Cannot get fullpath")
}
manifest, err := readPodManifest(fullPath + POD_MANIFEST)
if err != nil {
return nil, errors.Annotate(err, "Failed to read pod manifest")
}
fields := data.WithField("pod", manifest.Name.String())
target := path + PATH_TARGET
if cnt.Home.Config.TargetWorkDir != "" {
currentAbsDir, err := filepath.Abs(cnt.Home.Config.TargetWorkDir + "/" + manifest.Name.ShortName())
if err != nil {
logs.WithEF(err, fields).Panic("invalid target path")
}
target = currentAbsDir
}
pod := &Pod{
fields: fields,
path: fullPath,
args: args,
target: target,
manifest: *manifest,
}
return pod, nil
}
func IpLookup(host string, preferIPv4 bool) (net.IP, error) {
ips, err := net.LookupIP(host)
if err != nil || len(ips) == 0 || len(ips[0]) == 0 || len(ips[0]) != net.IPv6len {
return []byte{}, errs.WithEF(err, data.WithField("host", host), "Lookup failed or empty lookup result")
}
return processIPs(preferIPv4, ips)
}
func (e *EntryError) WithField(name string, value interface{}) *EntryError {
if e.Fields == nil {
e.Fields = data.WithField(name, value)
} else {
e.Fields = e.Fields.WithField(name, value)
}
return e
}
func (n *ReportSortType) UnmarshalJSON(d []byte) error {
var s string
if err := json.Unmarshal(d, &s); err != nil {
return errs.WithF(data.WithField("value", string(d)), "Failed to unmarsal serverSort")
}
switch strings.ToLower(s) {
case string(SORT_RANDOM):
*n = SORT_RANDOM
case string(SORT_NAME):
*n = SORT_NAME
case string(SORT_DATE):
*n = SORT_DATE
default:
return errs.WithF(data.WithField("value", s), "Unknown serverSort")
}
return nil
}
func ToAppcIsolators(isos []Isolator) (types.Isolators, error) {
isolators := types.Isolators{}
for _, i := range isos {
content, err := json.Marshal(i)
if err != nil {
return nil, errs.WithEF(err, data.WithField("isolator", i.Name), "Failed to marshall isolator")
}
isolator := types.Isolator{}
if err := isolator.UnmarshalJSON(content); err != nil {
return nil, errs.WithEF(err, data.WithField("isolator", i.Name), "Failed to unmarshall isolator")
}
isolators = append(isolators, isolator)
}
return isolators, nil
}
func (p *Port) UnmarshalJSON(b []byte) error {
var i int
if err := json.Unmarshal(b, &i); err == nil {
*p = Port(i)
return nil
}
var s string
if err := json.Unmarshal(b, &s); err == nil {
si, err := strconv.Atoi(s)
if err != nil {
return errs.WithEF(err, data.WithField("content", string(b)), "Invalid port value")
}
*p = Port(si)
return nil
} else {
return errs.WithEF(err, data.WithField("content", string(b)), "Failed to parse port")
}
}
func NewTemplateDir(path string, targetRoot string) *templateDir {
fields := data.WithField("dir", path)
logs.WithF(fields).Info("Reading template dir")
return &templateDir{
fields: fields,
src: path,
dst: targetRoot,
}
}
func NewTemplateDir(path string, targetRoot string) (*TemplateDir, error) {
fields := data.WithField("dir", path)
logs.WithF(fields).Info("Reading template dir")
tmplDir := &TemplateDir{
fields: fields,
src: path,
dst: targetRoot,
}
return tmplDir, tmplDir.LoadPartial()
}
func NewTemplateDir(path string, targetRoot string, continueOnError bool) (*TemplateDir, error) {
fields := data.WithField("dir", path).WithField("continueOnError", continueOnError)
logs.WithF(fields).Debug("Reading template dir")
tmplDir := &TemplateDir{
fields: fields,
src: path,
dst: targetRoot,
continueOnError: continueOnError,
}
return tmplDir, tmplDir.LoadPartial()
}
func NewPod(path string, args BuildArgs, checkWg *sync.WaitGroup) (*Pod, error) {
if (args.CatchOnError || args.CatchOnStep) && !args.SerialBuild {
args.SerialBuild = true
}
fullPath, err := filepath.Abs(path)
if err != nil {
logs.WithE(err).WithField("path", path).Fatal("Cannot get fullpath")
}
manifest, err := readPodManifest(fullPath + pathPodManifestYml)
if err != nil {
manifest2, err2 := readPodManifest(fullPath + "/cnt-pod-manifest.yml")
if err2 != nil {
return nil, errs.WithEF(err, data.WithField("path", fullPath+pathPodManifestYml).WithField("err2", err2), "Failed to read pod manifest")
}
logs.WithField("old", "cnt-pod-manifest.yml").WithField("new", "pod-manifest.yml").Warn("You are using the old aci configuration file")
manifest = manifest2
}
fields := data.WithField("pod", manifest.Name.String())
target := path + pathTarget
if Home.Config.TargetWorkDir != "" {
currentAbsDir, err := filepath.Abs(Home.Config.TargetWorkDir + "/" + manifest.Name.ShortName())
if err != nil {
logs.WithEF(err, fields).Panic("invalid target path")
}
target = currentAbsDir
}
pod := &Pod{
checkWg: checkWg,
fields: fields,
path: fullPath,
args: args,
target: target,
manifest: *manifest,
}
return pod, nil
}
func (b BuilderCommand) CommandManifestKey() (string, error) {
switch b {
case CommandBuild:
return "blablacar.github.io/dgr/stage1/build", nil
case CommandInit:
return "blablacar.github.io/dgr/stage1/init", nil
case CommandTry:
return "blablacar.github.io/dgr/stage1/try", nil
default:
return "", errs.WithF(data.WithField("command", b), "Unimplemented command manifest key")
}
}
func ExtractManifestFromAci(aciPath string) (*schema.ImageManifest, error) {
fields := data.WithField("file", aciPath)
content, err := ExtractManifestContentFromAci(aciPath)
if err != nil {
return nil, errs.WithEF(err, fields, "Cannot extract aci manifest content from file")
}
im := &schema.ImageManifest{}
if err = im.UnmarshalJSON(content); err != nil {
return nil, errs.WithEF(err, fields.WithField("content", string(content)), "Cannot unmarshall json content")
}
return im, nil
}
func NewAciWithManifest(path string, args BuildArgs, manifestTmpl string, checkWg *sync.WaitGroup) (*Aci, error) {
manifest, err := common.ProcessManifestTemplate(manifestTmpl, nil, false)
if err != nil {
return nil, errs.WithEF(err, data.WithField("content", manifestTmpl), "Failed to process manifest")
}
if manifest.NameAndVersion == "" {
logs.WithField("path", path).Fatal("name is mandatory in manifest")
}
fields := data.WithField("aci", manifest.NameAndVersion.String())
logs.WithF(fields).WithFields(data.Fields{"args": args, "path": path, "manifest": manifest}).Debug("New aci")
fullPath, err := filepath.Abs(path)
if err != nil {
return nil, errs.WithEF(err, fields, "Cannot get fullpath of project")
}
target := fullPath + pathTarget
if Home.Config.TargetWorkDir != "" {
currentAbsDir, err := filepath.Abs(Home.Config.TargetWorkDir + "/" + manifest.NameAndVersion.ShortName())
if err != nil {
return nil, errs.WithEF(err, fields.WithField("path", path), "Invalid target path")
}
target = currentAbsDir
}
aci := &Aci{
fields: fields,
args: args,
path: fullPath,
manifestTmpl: manifestTmpl,
manifest: manifest,
target: target,
FullyResolveDep: true,
checkWg: checkWg,
}
return aci, nil
}
func NewAttributesMerger(path string) (*AttributesMerger, error) {
in := newInputs(path)
// initialize input files list
err := in.listFiles()
if err != nil {
errs.WithEF(err, data.WithField("dir", path), "Cannot list files of dir")
}
res := []string{}
for _, file := range in.Files {
res = append(res, in.Directory+file)
}
return &AttributesMerger{dir: res}, nil
}