// ProcessHooks is intended for executing appropriate hooks.
// The hooks must contain the following prefix : [0-9]+_
// If arguments are being passed to the function, they will be
// passed tp the hooks as well
// Example:
// 01_pre_deploy
// 02_deploy
// 03_post_deploy
// 04_clean
func ProcessHooks(pathToHooksDir string, hookArgs ...string) error {
d, err := os.Stat(pathToHooksDir)
if err != nil {
return utils.FormatError(err)
}
if !d.IsDir() {
return utils.FormatError(fmt.Errorf("%s is not directory", d.Name()))
}
var scriptsSlice []string
err = filepath.Walk(pathToHooksDir, func(scriptName string, info os.FileInfo, err error) error {
if !info.IsDir() {
if found, _ := regexp.MatchString("[0-9]+_", scriptName); found {
scriptsSlice = append(scriptsSlice, scriptName)
}
}
return nil
})
sort.Strings(scriptsSlice)
for _, file := range scriptsSlice {
if err := exec.Command(file, hookArgs[0:]...).Run(); err != nil {
return utils.FormatError(err)
}
}
return nil
}
func (b *MetadataBuilder) Run() (deployer.Artifact, error) {
// in case no source template exists apparently we should use the default metadata
_, err := os.Stat(b.Source)
if err != nil {
b.Source = b.Dest
}
f, err := ioutil.ReadFile(b.Source)
if err != nil {
return nil, utils.FormatError(err)
}
data, err := utils.ProcessTemplate(string(f), b.UserData)
if err != nil {
return nil, utils.FormatError(err)
}
run := utils.RunFunc(b.SshConfig)
if _, err := run(fmt.Sprintf("echo \"%s\" > %s", data, b.Dest)); err != nil {
return nil, utils.FormatError(err)
}
return &deployer.CommonArtifact{
Name: filepath.Base(b.Dest),
Path: b.Dest,
Type: deployer.MetadataArtifact,
}, nil
}
// ImageCustomize treating image customization according to XML config files
// Returns error or nil
func Customize(pathToSlash, pathToConfigDir string) error {
//install/deinstall appropriate packages
pathToXml := pathToConfigDir + "/packages.xml"
if _, err := os.Stat(pathToXml); err == nil {
if err := packageManip(pathToXml, pathToSlash); err != nil {
return utils.FormatError(err)
}
}
// inject appropriate stuff
pathToXml = pathToConfigDir + "/inject_items.xml"
if _, err := os.Stat(pathToXml); err == nil {
if err := injectManip(pathToXml, pathToSlash); err != nil {
return utils.FormatError(err)
}
}
// services manipulation
pathToXml = pathToConfigDir + "/services.xml"
if _, err := os.Stat(pathToXml); err == nil {
if err := serviceManip(pathToXml, pathToSlash); err != nil {
return utils.FormatError(err)
}
}
// file content modification
pathToXml = pathToConfigDir + "/files_content.xml"
if _, err := os.Stat(pathToXml); err == nil {
if err := filesContentManip(pathToXml, pathToSlash); err != nil {
return utils.FormatError(err)
}
}
return nil
}
// Emulator returns appropriate path to QEMU emulator for a given architecture
func (d *Driver) Emulator(arch string) (string, error) {
switch arch {
case "x86_64":
case "i686":
default:
return "", utils.FormatError(fmt.Errorf("Unsupported architecture(%s).Only i686 and x86_64 supported", arch))
}
out, err := d.Run("virsh capabilities")
if err != nil {
return "", utils.FormatError(err)
}
m, err := mxj.NewMapXml([]byte(out))
if err != nil {
return "", utils.FormatError(err)
}
v, _ := m.ValuesForPath("capabilities.guest.arch", "-name:"+arch)
// fixing https://github.com/dorzheh/deployer/issues/2
if len(v) == 0 {
return "", utils.FormatError(fmt.Errorf("Can't gather a KVM guest information for architecture %s.", arch))
}
return v[0].(map[string]interface{})["emulator"].(string), nil
}
func CreateConfig(d *deployer.CommonData, i *metadata.InputData) (*metadata.Config, error) {
if d.DefaultExportDir == "" {
d.DefaultExportDir = "/var/lib/libvirt/images"
}
m, err := metadata.NewMetdataConfig(d, i.StorageConfigFile)
if err != nil {
return nil, utils.FormatError(err)
}
controller.RegisterSteps(func() func() error {
return func() error {
var err error
m.Hwdriver, err = hwinfodriver.NewHostinfoDriver(m.SshConfig, i.Lshw, filepath.Join(d.RootDir, ".hwinfo.json"))
if err != nil {
return utils.FormatError(err)
}
m.Metadata = new(metadata.Metadata)
m.EnvDriver = envdriver.NewDriver(m.SshConfig)
if m.Metadata.EmulatorPath, err = m.EnvDriver.Emulator(d.Arch); err != nil {
return utils.FormatError(err)
}
return controller.SkipStep
}
}())
if err := metadata.RegisterSteps(d, i, m, &meta{}); err != nil {
return nil, utils.FormatError(err)
}
return m, nil
}
// Deploy is implementing entire flow
// The flow consists of the following stages:
// - CreateConfig creates appropriate configuration(user interaction against UI).
// - CreateBuilders creates appropriate builders and passes them to the build process
// - CreatePostProcessors creates appropriate post-processors and passes them for post-processing
func Deploy(c *deployer.CommonData, f deployer.FlowCreator) error {
if err := f.CreateConfig(c); err != nil {
return utils.FormatError(err)
}
builders, err := f.CreateBuilders(c)
if err != nil {
return utils.FormatError(err)
}
artifacts, err := deployer.BuildProgress(c, builders)
if err != nil {
return utils.FormatError(err)
}
post, err := f.CreatePostProcessor(c)
if err != nil {
return utils.FormatError(err)
}
if post != nil {
err := deployer.PostProcessProgress(c, post, artifacts)
if err != nil {
return utils.FormatError(err)
}
}
return nil
}
// Build iterates over a slice of builders and runs
// each builder in a separated goroutine.
// Returns a slice of artifacts.
func Build(builders []Builder) ([]Artifact, error) {
dur, err := time.ParseDuration("1s")
if err != nil {
return nil, utils.FormatError(err)
}
runtime.GOMAXPROCS(runtime.NumCPU() - 1)
var artifacts []Artifact
ch := make(chan *buildResult, len(builders))
for _, b := range builders {
time.Sleep(dur)
go func(b Builder) {
artifact, err := b.Run()
// Forwards created artifact to the channel.
ch <- &buildResult{artifact, err}
}(b)
}
for i := 0; i < len(builders); i++ {
select {
case result := <-ch:
if result.err != nil {
//defer close(ch)
return nil, utils.FormatError(result.err)
}
artifacts = append(artifacts, result.artifact)
}
}
return artifacts, nil
}
func ProcessNetworkTemplate(mode *xmlinput.Mode, defaultTemplate string, tmpltData interface{}, templatesDir string) (string, error) {
var customTemplate string
if mode.Tmplt == nil {
customTemplate = defaultTemplate
} else {
var templatePath string
if templatesDir != "" {
templatePath = filepath.Join(templatesDir, mode.Tmplt.FileName)
} else {
templatePath = filepath.Join(mode.Tmplt.Dir, mode.Tmplt.FileName)
}
buf, err := ioutil.ReadFile(templatePath)
if err != nil {
return "", utils.FormatError(err)
}
customTemplate = string(buf)
}
tempData, err := utils.ProcessTemplate(customTemplate, tmpltData)
if err != nil {
return "", utils.FormatError(err)
}
return string(tempData) + "\n", nil
}
func verify(data *XMLInputData) error {
if data.Networks.Configure {
for _, net := range data.Networks.Configs {
seenDirect := false
seenPassthrough := false
for _, mode := range net.Modes {
switch mode.Type {
case ConTypeDirect:
seenDirect = true
case ConTypePassthrough:
seenPassthrough = true
case ConTypeBridged, ConTypeOVS, ConTypeVirtualNetwork, ConTypeSRIOV:
default:
return utils.FormatError(errors.New("unexpected mode " + string(mode.Type)))
}
}
if seenDirect && seenPassthrough {
return utils.FormatError(errors.New("either \"direct\" or\"passthrough\" permitted"))
}
}
for _, nic := range data.HostNics.Allowed {
if nic.Disjunction && nic.Priority {
return utils.FormatError(errors.New("either \"priority\" or\"disjunction\" permitted"))
}
}
}
return nil
}
// getMappers is responsible for finding mappers bound to appropriate loop device
// and providing the stuff as a slice
func (i *image) getMappers(loopDeviceName string) ([]string, error) {
var mappers []string
if _, err := i.run(i.utils.Kpartx + " -a " + loopDeviceName); err != nil {
return mappers, utils.FormatError(err)
}
// somehow on RHEL based systems refresh might take some time therefore
// no mappers are available until then
duration, err := time.ParseDuration("1s")
if err != nil {
return mappers, utils.FormatError(err)
}
time.Sleep(duration)
cmd := fmt.Sprintf("find /dev -name loop%sp[0-9]",
strings.TrimSpace(strings.SplitAfter(loopDeviceName, "/dev/loop")[1]))
out, err := i.run(cmd)
if err != nil {
return mappers, utils.FormatError(fmt.Errorf("%s [%v]", out, err))
}
for _, line := range strings.Split(out, "\n") {
if line != "" {
mappers = append(mappers, line)
}
}
if len(mappers) == 0 {
return mappers, utils.FormatError(errors.New("mappers not found"))
}
sort.Strings(mappers)
return mappers, nil
}
// addMapper registers appropriate mapper and it's mount point
func (i *image) addMapper(mapperDeviceName, path string) error {
mountPoint := filepath.Join(i.slashpath, path)
if out, err := i.run("mkdir -p " + mountPoint); err != nil {
return utils.FormatError(fmt.Errorf("%s [%v]", out, err))
}
// check if the volume is already mounted
mounted, err := isMounted(mapperDeviceName)
if err != nil {
return utils.FormatError(err)
}
if !mounted {
if out, err := i.run(fmt.Sprintf("mount %s %s", mapperDeviceName, mountPoint)); err != nil {
return utils.FormatError(fmt.Errorf("%s [%v]", out, err))
}
}
// add mapper
i.mappers = append(i.mappers,
&mapperDevice{
name: mapperDeviceName,
mountPoint: mountPoint,
},
)
// advance amount of mappers
i.loopDevice.amountOfMappers++
return nil
}
// ParseXMLInputFile is responsible for parsing a given configuration file
// representing appropriate input data to a srtuctured form
func ParseXMLInput(config string) (*XMLInputData, error) {
d, err := utils.ParseXMLFile(config, new(XMLInputData))
if err != nil {
return nil, utils.FormatError(err)
}
if err := verify(d.(*XMLInputData)); err != nil {
return nil, utils.FormatError(err)
}
return d.(*XMLInputData), nil
}
// ParseXMLInputBuf is responsible for parsing a given stream of bytes
// representing appropriate input data to a srtuctured form
func ParseXMLInputBuf(data []byte) (*XMLInputData, error) {
d, err := utils.ParseXMLBuff(data, new(XMLInputData))
if err != nil {
return nil, utils.FormatError(err)
}
if err := verify(d.(*XMLInputData)); err != nil {
return nil, utils.FormatError(err)
}
return d.(*XMLInputData), nil
}
func NewParserBuff(bundleConfigStream []byte, s BundleStrategy) (*Parser, error) {
if s == nil {
return nil, utils.FormatError(errors.New("bundle strategy is nil"))
}
data, err := utils.ParseXMLBuff(bundleConfigStream, s)
if err != nil {
return nil, utils.FormatError(err)
}
return &Parser{s, data}, nil
}
func NewParser(bundleConfigFile string, s BundleStrategy) (*Parser, error) {
if s == nil {
return nil, utils.FormatError(errors.New("bundle strategy is nil"))
}
data, err := utils.ParseXMLFile(bundleConfigFile, s)
if err != nil {
return nil, utils.FormatError(err)
}
return &Parser{s, data}, nil
}
// Parse parses lshw output
func (c *Collector) Hwinfo2Json() error {
lshwNewPath, err := c.prepare()
if err != nil {
return utils.FormatError(err)
}
out, err := c.Run(lshwNewPath + " -class network -class cpu -json")
if err != nil {
return utils.FormatError(err)
}
return ioutil.WriteFile(c.hwinfoFile, []byte(out), 0)
}
func (b *InstanceBuilder) Run() (a deployer.Artifact, err error) {
if err = b.Filler.CustomizeRootfs("/"); err != nil {
err = utils.FormatError(err)
return
}
if err = b.Filler.InstallApp("/"); err != nil {
err = utils.FormatError(err)
return
}
return
}
func (c *Collector) CPUs() (int, error) {
cpustr, err := c.Run(`grep -c ^processor /proc/cpuinfo`)
if err != nil {
return 0, utils.FormatError(err)
}
cpus, err := strconv.Atoi(strings.Trim(cpustr, "\n"))
if err != nil {
return 0, utils.FormatError(err)
}
return int(cpus), nil
}
// filesContentManip manipulates with the content of the files
// according to appropriate XML configuration file
// Example:
//<files>
// <file>
// <path>/etc/sysconfig/selinux</path>
// <bkp_name>/etc/sysconfig/selinux.bkp</bkp_name>
// <action>replace</action>
// <old_pattern>SELINUX=\S+</old_pattern>
// <new_pattern>SELINUX=disabled</new_pattern>
// </file>
// <file>
// <path>/etc/passwd</path>
// <bkp_name>/etc/passwd.bak</bkp_name>
// <action>append</action>
// <old_pattern></old_pattern>
// <new_pattern>test:x:111:111::/root:/bin/bash</new_pattern>
// </file>
//</files>
func filesContentManip(pathToXml, pathToSlash string) error {
dataBuf, err := ioutil.ReadFile(pathToXml)
if err != nil {
return utils.FormatError(err)
}
fileContentStruct := FilesContent{}
if err := xml.Unmarshal(dataBuf, &fileContentStruct); err != nil {
return utils.FormatError(err)
}
for _, val := range fileContentStruct.FContent {
targetPath := filepath.Join(pathToSlash, val.Path)
if err != nil {
return utils.FormatError(err)
}
finfo, err := os.Stat(targetPath)
if err != nil {
continue
}
if val.NewPattern == "" {
return utils.FormatError(errors.New("configuration error - NewPattern is empty"))
}
if val.BkpName != "" {
bkpFilePath := filepath.Join(pathToSlash, val.BkpName)
if err := ioutils.CopyFile(targetPath, bkpFilePath, 0, -1, -1, false); err != nil {
return utils.FormatError(err)
}
}
fd, err := os.OpenFile(targetPath, os.O_RDWR|os.O_APPEND, finfo.Mode())
if err != nil {
return utils.FormatError(err)
}
defer fd.Close()
switch val.Action {
// if we need to append to the file
case ACTION_APPEND:
if err := ioutils.AppendToFd(fd, val.NewPattern+"\n", val.NewPattern); err != nil {
return utils.FormatError(err)
}
// if we need to replace a pattern
case ACTION_REPLACE:
if val.OldPattern == "" {
return utils.FormatError(errors.New("configuration error - replace action is set but OldPattern is empty"))
}
if err := ioutils.FindAndReplaceFd(fd, val.OldPattern, val.NewPattern); err != nil {
return utils.FormatError(err)
}
default:
return utils.FormatError(errors.New(`FilesContentManip:configuration error - unsupported action`))
}
}
return nil
}
func (i *image) bind(imagePath string) (loopDevice string, err error) {
loopDevice, err = i.run("losetup -f")
if err != nil {
err = utils.FormatError(err)
return
}
cmd := fmt.Sprintf("losetup %s %s", loopDevice, imagePath)
if out, er := i.run(cmd); err != nil {
err = utils.FormatError(fmt.Errorf("%s [%v]", out, er))
return
}
return
}
// convert is responsible for converting RAW image to other format
func (i *image) convert() error {
// set the new path - append extention
newPath := fmt.Sprintf("%s.%s", strings.TrimSuffix(i.config.Path, ".raw"), i.config.Type)
if out, err := i.run(fmt.Sprintf("qemu-img convert -f raw -O %s %s %s", i.config.Type, i.config.Path, newPath)); err != nil {
return utils.FormatError(fmt.Errorf("%s [%v]", out, err))
}
//remove temporary image
if out, err := i.run("rm -rf " + i.config.Path); err != nil {
return utils.FormatError(fmt.Errorf("%s [%v]", out, err))
}
// expose the new path
i.config.Path = newPath
return nil
}
func (b *ImageBuilder) Run() (deployer.Artifact, error) {
if err := os.MkdirAll(b.RootfsMp, 0755); err != nil {
return nil, utils.FormatError(err)
}
defer os.RemoveAll(b.RootfsMp)
// create new image artifact
img, err := image.New(b.ImageConfig, b.RootfsMp, b.Utils, b.SshfsConfig)
if err != nil {
return nil, utils.FormatError(err)
}
// interrupt handler
img.ReleaseOnInterrupt()
defer func() {
img.Cleanup()
}()
// parse the image
if err := img.Parse(); err != nil {
return nil, utils.FormatError(err)
}
// customize rootfs
if b.Filler != nil {
if err := b.Filler.CustomizeRootfs(b.RootfsMp); err != nil {
return nil, utils.FormatError(err)
}
// install application.
if err := b.Filler.InstallApp(b.RootfsMp); err != nil {
return nil, utils.FormatError(err)
}
}
if b.ImageConfig.Bootable {
if err := img.MakeBootable(); err != nil {
return nil, utils.FormatError(err)
}
}
if b.Filler != nil {
if err := b.Filler.RunHooks(b.RootfsMp); err != nil {
return nil, utils.FormatError(err)
}
}
if err := img.Cleanup(); err != nil {
return nil, utils.FormatError(err)
}
if err := img.Convert(); err != nil {
return nil, utils.FormatError(err)
}
return &deployer.CommonArtifact{
Name: filepath.Base(b.ImageConfig.Path),
Path: b.ImageConfig.Path,
Type: deployer.ImageArtifact,
}, nil
}
func (b *DirBuilder) Run() (a deployer.Artifact, err error) {
// customize rootfs
if b.Filler != nil {
if err = b.Filler.CustomizeRootfs(b.RootfsPath); err != nil {
err = utils.FormatError(err)
return
}
// install application
if err = b.Filler.InstallApp(b.RootfsPath); err != nil {
err = utils.FormatError(err)
return
}
}
return
}
// numa4Nic fetchs NUMA node for appropriate PCI device
func (c *Collector) numa4Nic(pciAddr string) (int, error) {
numaInt := 0
numaStr, err := c.Run("cat /sys/bus/pci/devices/" + pciAddr + "/numa_node")
if err != nil {
return numaInt, utils.FormatError(err)
}
if numaStr == "-1" {
numaInt = 0
} else {
numaInt, err = strconv.Atoi(numaStr)
if err != nil {
return numaInt, utils.FormatError(err)
}
}
return numaInt, nil
}
func UiSelectEnv(c *deployer.CommonData, envList []string, envs []deployer.FlowCreator) error {
var menuList []string
indexInt := 1
for _, env := range envList {
menuList = append(menuList, strconv.Itoa(indexInt), env)
indexInt++
}
envsNum := len(envs)
dType := 0
if envsNum > 1 {
c.Ui.SetTitle("Select environment")
c.Ui.SetSize(envsNum+7, 30)
resStr, err := c.Ui.Menu(envsNum, menuList[0:]...)
if err != nil && err.Error() == gui.DialogExit {
os.Exit(0)
}
dType, err = strconv.Atoi(resStr)
if err != nil {
return utils.FormatError(err)
}
dType--
}
return main.Deploy(c, envs[dType])
}
// Destroy is responsible for removing appropriate artifact.
func (a *CommonArtifact) Destroy() error {
run := utils.RunFunc(a.SshConfig)
if _, err := run("rm " + a.Path); err != nil {
return utils.FormatError(err)
}
return nil
}
// ParseConfigFile is responsible for reading appropriate XML file
// and calling ParseConfig for further processing
func ParseConfigFile(xmlpath string) (*Storage, error) {
fb, err := ioutil.ReadFile(xmlpath)
if err != nil {
return nil, utils.FormatError(err)
}
return ParseConfig(fb)
}
// DiskConfig returns disk configuration for appropriate index
func (s *Storage) IndexToConfig(index ConfigIndex) (*Config, error) {
t := s.Configs[index]
if t == nil {
return nil, utils.FormatError(fmt.Errorf("no configuration found for index %d", index))
}
return t, nil
}
// Parse processes RAW image
// Returns error/nil
func (i *image) Parse() error {
var err error
if i.loopDevice.name, err = i.bind(i.config.Path); err != nil {
return utils.FormatError(err)
}
if i.needToFormat {
if err := i.partTableMakefs(); err != nil {
return utils.FormatError(err)
}
} else if i.config.Partitions != nil {
if err := i.addMappers(); err != nil {
return utils.FormatError(err)
}
}
return nil
}
// create is intended for creating RAW image
func (i *image) create() error {
out, err := i.run(fmt.Sprintf("dd if=/dev/zero of=%s count=1 bs=1 seek=%vM", i.config.Path, i.config.SizeMb))
if err != nil {
return utils.FormatError(fmt.Errorf("%s [%v]", out, err))
}
return nil
}