func Serve(log *logging.Logger, addr string, ssl *ini.File) {
var cmd []byte
var ln net.Listener
if ssl != nil {
certificate := ssl.Section("ssl").Key("certificate").String()
key := ssl.Section("ssl").Key("key").String()
ln = tlsServe(log, addr, certificate, key)
log.Debug("Opened SSL socket")
} else {
ln = plainServe(log, addr)
log.Debug("Opened plain socket")
}
defer ln.Close()
for {
conn, err := ln.Accept()
buff := bufio.NewReader(conn)
if err != nil {
// handle error
log.Error("Error: %v\n", err)
}
log.Debug("Connection from " + conn.RemoteAddr().String() + " accepted")
cmd, err = buff.ReadBytes('\n')
log.Debug("Remote data readed")
Parse(log, cmd, conn)
conn.Close()
}
}
func fileBackup(log *logging.Logger, section *common.Section, cfg *ini.File, c chan bool, wg *sync.WaitGroup) {
defer func() {
<-c
wg.Done()
}()
dataset.DelDataset(log, cfg, section.Name, section.Grace, section.Dataset)
log.Info("About to backup section " + section.Name)
// Execute pre_command
if err := execCommand(log, cfg.Section(section.Name), "pre_command"); err != nil {
log.Debug(err.Error())
// TODO: manage error
}
// Backup!
backup.Filebackup(log, section, cfg)
// Execute post_command
if err := execCommand(log, cfg.Section(section.Name), "post_command"); err != nil {
log.Debug(err.Error())
// TODO: manage error
}
}
func Init() {
cfgFile := getAbs("./settings/settings.ini")
cfg := new(ini.File)
cfg.BlockMode = false
cfg, err := ini.Load(cfgFile)
if err != nil {
panic(err)
}
cfg.MapTo(&settingStruct)
settingStruct.Log.File = filepath.Join(
getAbs(settingStruct.Log.Path),
time.Now().Format(settingStruct.Log.Format),
)
//map to global
{
Static = settingStruct.Static
Server = settingStruct.Server
Filesync = settingStruct.Filesync
Template = settingStruct.Template
DefaultVars = settingStruct.DefaultVars
Admin = settingStruct.Admin
Log = settingStruct.Log
}
FsCfgMgr = new(cfgMgr)
FsCfgMgr.Init()
go watch()
}
func fsGetData(log *logging.Logger, section *common.Section, cfg *ini.File) {
var previous int
var res common.JSONFile
var db database.DB
if section.Dataset-1 == 0 {
previous = cfg.Section("dataset").Key(section.Grace).MustInt()
} else {
previous = section.Dataset - 1
}
db.Open(log, cfg)
defer db.Close()
for _, item := range []string{"directory", "file", "symlink"} {
for res = range database.ListItems(log, &db, section, item) {
switch item {
case "directory":
fsSaveData(log, cfg, section, res, false)
case "symlink":
fsSaveData(log, cfg, section, res, false)
case "file":
if database.ItemExist(log, &db, &res, section, previous) {
fsSaveData(log, cfg, section, res, true)
} else {
fsSaveData(log, cfg, section, res, false)
}
}
}
}
}
func fileRestore(log *logging.Logger, cfg *ini.File, section *common.Section) {
// Execute pre_command
execCommand(log, cfg.Section(section.Name), "pre_command")
// Restore!
restore.FileRestore(log, section, cfg)
// Execute post_command
execCommand(log, cfg.Section(section.Name), "post_command")
}
func Backup(log *logging.Logger, cfg *ini.File, grace string, reload bool) {
const POOL = 5
var db database.DB
var tx *sql.Tx
var c = make(chan bool, POOL)
var wg = new(sync.WaitGroup)
var dataset, maxdatasets int
var sections []*ini.Section
sections = cfg.Sections()
maxdatasets, _ = cfg.Section("dataset").Key(grace).Int()
db.Open(log, cfg)
defer db.Close()
tx, _ = db.Conn.Begin()
dataset = database.GetDataset(log, tx, grace)
tx.Commit()
if !reload {
if nextds := dataset + 1; nextds > maxdatasets {
dataset = 1
} else {
dataset = dataset + 1
}
}
log.Info("Dataset processed: " + strconv.Itoa(dataset))
wg.Add(len(sections) - len(SECT_RESERVED))
for _, section := range sections {
if !contains(SECT_RESERVED, section.Name()) {
if section.Key("type").String() == "file" { // FIXME: useless?
sect := common.Section{
Name: section.Name(),
Grace: grace,
Dataset: dataset,
Compressed: section.Key("compress").MustBool(),
}
go fileBackup(log, §, cfg, c, wg)
c <- true
}
}
}
wg.Wait() // Wait for all the children to die
close(c)
tx, _ = db.Conn.Begin()
database.SetDataset(log, tx, dataset, grace)
tx.Commit()
}
func handleServices(confIni *ini.File, p *proxy.IsolationProxy) {
confSection, err := confIni.GetSection("services")
check(err)
for _, service := range confSection.Keys() {
if !isRunningLocally(service.Name(), service.String(), p.Registry) {
go func(service *ini.Key) {
check(p.Handle(service.Name(), service.String()))
}(service)
} else {
// If the service should be running locally on the same port, don't proxy it
services.L.Warnf("Not handling %s on %s: should be running locally", service.Name(), service.String())
}
}
}
func (db *DB) retrieveCfgData(cfg *ini.File) {
db.User = cfg.Section("database").Key("user").String()
db.Password = cfg.Section("database").Key("password").String()
db.Host = cfg.Section("database").Key("host").String()
db.Port = cfg.Section("database").Key("port").String()
db.Database = cfg.Section("database").Key("dbname").String()
}
func (d *serverConf) generateConnInfoOfUser(ii *ini.File, user string) error {
u, err := d.AuthSys.UserInfo(user)
if err != nil {
return err
}
keyBytes, err := MarshalPublicKey(d.publicKey)
if err != nil {
return err
}
url := fmt.Sprintf("d5://%s:%s@%s/%s=%s/%s", u.Name, u.Pass, d.Listen, d.ServerName, NameOfKey(d.publicKey), d.Cipher)
sec, _ := ii.NewSection(CF_CREDENTIAL)
sec.NewKey(CF_URL, url)
sec.NewKey(CF_KEY, base64.StdEncoding.EncodeToString(keyBytes))
sec.Comment = _COMMENTED_PAC_SECTION
return nil
}
func Restore(log *logging.Logger, cfg *ini.File, grace string) {
dataset := cfg.Section("general").Key("dataset").MustInt()
for _, section := range cfg.Sections() {
if !contains(SECT_RESERVED, section.Name()) {
if section.Key("type").String() == "file" {
sect := common.Section{
Name: section.Name(),
Grace: grace,
Dataset: dataset,
Compressed: section.Key("compress").MustBool(),
}
fileRestore(log, cfg, §)
}
}
}
}
// Loads the primary configuration file and maps it into a Config type.
func parseConfigINI(cfgPathAbs string) (*Config, error) {
var err error
var cfg *Config = new(Config)
var iniCfg *ini.File
// loading the INI file contents into local struct
if iniCfg, err = ini.Load(cfgPathAbs); err != nil {
log.Fatalln("[Config] Errors on parsing INI file:", err)
return nil, err
}
// mapping configuration into local struct
if err = iniCfg.MapTo(cfg); err != nil {
log.Fatalln("[Config] Errors on mapping INI:", err)
return nil, err
}
return cfg, nil
}
func FileRestore(log *logging.Logger, section *common.Section, cfg *ini.File) {
var cmd common.JSONMessage
var res common.JSONFile
var db database.DB
var err error
db.Open(log, cfg)
defer db.Close()
cmd.Context = "file"
cmd.Command.Name = "put"
if cfg.Section(section.Name).Key("path").String() == "" {
log.Debug("About to full restore")
cmd.Command.ACL = cfg.Section(section.Name).Key("acl").MustBool()
for _, item := range []string{"directory", "file", "symlink"} {
for res = range database.ListItems(log, &db, section, item) {
if cmd.Command.ACL {
log.Debug("About to restore ACLs for " + res.Name)
res.Acl, err = database.GetAcls(log, &db, res.Name, section)
if err != nil {
log.Error("ACLs extraction error for " + res.Name)
log.Debug("error: " + err.Error())
}
}
cmd.Command.Element = res
put(log, section, cfg, &cmd)
}
}
} else {
log.Debug("About to restore path " + cfg.Section(section.Name).Key("path").String())
cmd.Command.Element, err = database.GetItem(log, &db, cfg.Section(section.Name).Key("path").String(), section)
if err != nil {
log.Error("Error when putting data to section " + section.Name)
log.Debug("error: " + err.Error())
} else {
cmd.Command.ACL = cfg.Section(section.Name).Key("acl").MustBool()
put(log, section, cfg, &cmd)
}
}
}
func loadServiceRegistry(confIni *ini.File) proxy.LocalRegistry {
reg := make(proxy.LocalRegistry)
addService := func(name string, endpoints ...string) {
for _, addr := range endpoints {
endpoint := &proxy.Endpoint{
Service: name,
Host: addr,
}
endpoint.TestActive()
reg.Add(name, endpoint)
}
}
confSection, err := confIni.GetSection("backends")
check(err)
for _, service := range confSection.Keys() {
addService(service.Name(), service.Strings(",")...)
}
return reg
}
func put(log *logging.Logger, section *common.Section, cfg *ini.File, cmd *common.JSONMessage) {
var conn net.Conn
var err error
conn, err = network.GetSocket(cfg.Section(section.Name))
if err != nil {
log.Error("Error when opening connection with section " + section.Name)
log.Debug("error: " + err.Error())
return
}
defer conn.Close()
cmd.Send(conn)
if cmd.Command.Element.Type == "file" {
transfered := dataset.Path(cfg, section, false) + string(filepath.Separator) + cmd.Command.Element.Name
if err := common.SendFile(transfered, conn); err != nil {
log.Debug("Error when sending file: ", err.Error())
}
}
}
func Path(cfg *ini.File, section *common.Section, previous bool) string {
var dataset int
if previous {
if section.Dataset == 1 {
dataset = cfg.Section("dataset").Key(section.Grace).MustInt()
} else {
dataset = section.Dataset - 1
}
} else {
dataset = section.Dataset
}
destination := strings.Join([]string{
cfg.Section("general").Key("repository").String(),
section.Grace,
strconv.Itoa(dataset),
section.Name,
}, string(filepath.Separator))
return destination
}
func main() {
var cfg *ini.File
var backup bool
var reload bool
var err error
s := options.NewOptions(SPEC)
// Check if options isn't passed
if len(os.Args[1:]) <= 0 {
s.PrintUsageAndExit("No option specified")
}
opts := s.Parse(os.Args[1:])
grace := ""
// Print version and exit
if opts.GetBool("version") {
fmt.Println("Memento server " + VERSION)
os.Exit(0)
}
// Print help and exit
if opts.GetBool("help") {
s.PrintUsageAndExit("Memento server " + VERSION)
}
// Check backup or restore operation (mandatory)
if opts.GetBool("backup") && opts.GetBool("restore") {
// If backup and restore options are passed in the same session, print help and exit
s.PrintUsageAndExit("Cannot perform a backup and restore operations in the same session")
}
// Read grace (mandatory)
if opts.GetBool("hour") {
grace = "hour"
} else if opts.GetBool("day") {
grace = "day"
} else if opts.GetBool("week") {
grace = "week"
} else if opts.GetBool("month") {
grace = "month"
} else {
// If grace is not selected, print help and exit
s.PrintUsageAndExit("No grace selected")
}
if opts.GetBool("reload-dataset") {
reload = true
} else {
reload = false
}
if opts.GetBool("backup") {
backup = true
cfg, err = ini.Load([]byte{}, opts.Get("backup"))
} else if opts.GetBool("restore") {
backup = false
cfg, err = ini.Load([]byte{}, opts.Get("restore"))
}
if err != nil {
fmt.Println("Error about reading config file:", err)
os.Exit(1)
}
repository := cfg.Section("general").Key("repository").String()
checkStructure(repository)
loglevel, _ := logging.LogLevel(cfg.Section("general").Key("log_level").String())
log := setLog(loglevel, cfg.Section("general").Key("log_file").String())
log.Info("Started version " + VERSION)
log.Debug("Grace selected: " + grace)
if backup {
server.Backup(log, cfg, grace, reload)
} else {
server.Restore(log, cfg, grace)
}
log.Info("Ended version " + VERSION)
}
func fsSaveData(log *logging.Logger, cfg *ini.File, section *common.Section, data common.JSONFile, previous bool) {
var item, source, dest, hash string
var cmd common.JSONMessage
var conn net.Conn
var err error
item = dataset.ConvertPath(data.Os, data.Name)
source = dataset.Path(cfg, section, true) + string(filepath.Separator) + item
dest = dataset.Path(cfg, section, false) + string(filepath.Separator) + item
log.Debug("Save item: " + dest)
switch data.Type {
case "directory":
os.MkdirAll(dest, 0755)
case "symlink":
if err := os.Symlink(data.Link, dest); err != nil {
log.Error("Error when creating symlink for file %s", data.Name)
log.Debug("Trace: %s", err.Error())
}
case "file":
if previous {
if section.Compressed {
source = source + ".compressed"
dest = dest + ".compressed"
}
if err = os.Link(source, dest); err != nil {
log.Error("Error when link file %s", data.Name)
log.Debug("Trace: " + err.Error())
}
} else {
cmd.Context = "file"
cmd.Command.Name = "get"
cmd.Command.Element = data
conn, err = network.GetSocket(cfg.Section(section.Name))
if err != nil {
log.Error("Error when opening connection with section " + section.Name)
log.Debug("Trace: " + err.Error())
return
}
defer conn.Close()
cmd.Send(conn)
if hash, err = common.ReceiveFile(dest, conn); err != nil {
log.Error("Error when receiving file " + data.Name)
log.Debug("Trace: " + err.Error())
}
// TODO: check file's hash
if hash == "" {
log.Error("Hash for file " + dest + " mismatch")
// TODO: remove file if hash mismatch
} else {
log.Debug("Hash for file " + dest + " is " + hash)
if section.Compressed {
dataset.CompressFile(log, dest)
}
}
}
}
}
func fsGetMetadata(log *logging.Logger, section *common.Section, cfg *ini.File) {
var conn net.Conn
var cmd common.JSONMessage
var db database.DB
var buff *bufio.Reader
var res common.JSONResult
var tx *sql.Tx
var data []byte
var err error
log.Debug("Getting metadata for " + section.Name)
cmd.Context = "file"
cmd.Command.Name = "list"
cmd.Command.Paths = strings.Split(cfg.Section(section.Name).Key("path").String(), ",")
cmd.Command.ACL = cfg.Section(section.Name).Key("acl").MustBool()
cmd.Command.Exclude = cfg.Section(section.Name).Key("exclude").String()
log.Debug("Metadata command request: %+v", cmd)
conn, err = network.GetSocket(cfg.Section(section.Name))
if err != nil {
log.Error("Error when opening connection with section " + section.Name)
log.Debug("error: " + err.Error())
return
}
defer conn.Close()
cmd.Send(conn)
db.Open(log, cfg)
defer db.Close()
tx, err = db.Conn.Begin()
if err != nil {
log.Error("Transaction error for section " + section.Name)
log.Debug("Trace: " + err.Error())
return
}
buff = bufio.NewReader(conn)
for {
res = common.JSONResult{}
data, err = buff.ReadBytes('\n')
if err != nil {
if err == io.EOF {
log.Debug("All files's metadata are saved")
break
}
log.Error("Error when getting files metadata for section " + section.Name)
log.Debug("error: " + err.Error())
return
}
err = json.Unmarshal(data, &res)
if err != nil {
log.Error("Error when parsing JSON data for section " + section.Name)
log.Debug("error: " + err.Error())
return
}
log.Debug("Metadata received: %+v", res)
if err = database.SaveAttrs(log, tx, section, res.Data); err != nil {
log.Error("Failed saving database item: " + res.Data.Name)
log.Debug("Trace: " + err.Error())
tx.Rollback()
return
}
if err = database.SaveAcls(log, tx, section, res.Data.Name, res.Data.Acl); err != nil {
log.Error("Failed saving ACLs into database for item: " + res.Data.Name)
log.Debug("Trace: " + err.Error())
tx.Rollback()
return
}
}
tx.Commit()
}
// Receives a list of Container INI configuration files and load them into
// "Container" section of primary Config instance.
func (cfg *Config) loadIniConfigs(cfgPaths []string) error {
var err error
var iniCfg *ini.File
var section *ini.Section
var sectionName string
var cfgPath string
var serviceCfg *ServiceConfig
var containerCfg *ContainerConfig
var name string
var match bool
for _, cfgPath = range cfgPaths {
log.Printf("[Config] Loading: '%s'", cfgPath)
// avoiding dummy files
if match, _ = path.Match("\\.\\#*\\.ini", path.Base(cfgPath)); match {
log.Printf("[Config] Ignoring config file: '%s'", cfgPath)
continue
}
if iniCfg, err = ini.Load(cfgPath); err != nil {
log.Println("[Config] Config load error:", err)
return err
}
// the section names will determine witch kind of configuratio this is
for _, sectionName = range iniCfg.SectionStrings() {
section = iniCfg.Section(sectionName)
switch sectionName {
case "Service":
serviceCfg = new(ServiceConfig)
if err = section.MapTo(serviceCfg); err != nil {
log.Println("[Config] Error on mapTo ServiceConfig:", err)
return err
}
if name, err = sanitizeName(serviceCfg.Name); err != nil {
log.Println("[Config] Error on sanitize name:", err)
return err
}
log.Printf("[Config] Adding service: '%s'", name)
serviceCfg.Name = name
cfg.Service[name] = serviceCfg
log.Printf("[Config] DEBUG serviceCfg: '%+v'", serviceCfg)
case "Container":
containerCfg = new(ContainerConfig)
if err = section.MapTo(containerCfg); err != nil {
log.Println("[Config] Error on mapTo Container:", err)
return err
}
if name, err = sanitizeName(containerCfg.Name); err != nil {
log.Println("[Config] Error on sanitize name:", err)
return err
}
log.Printf("[Config] Adding container: '%s'", name)
containerCfg.Name = name
cfg.Container[name] = containerCfg
log.Printf("[Config] DEBUG containerCfg: '%+v'", containerCfg)
case "DEFAULT":
continue
default:
log.Printf("[Config] Ignored section: '%s'", sectionName)
}
}
}
return nil
}
func getSingleHostname(file string, section string) string {
var ini_cfg *ini.File
loginfo := "getSingleHostname"
hostname := ""
ini_cfg, err := ini.Load(file)
if err != nil || ini_cfg == nil {
fmt.Printf("[ERROR] %s() - %s.\n", loginfo, err)
return hostname
}
items, err := ini_cfg.GetSection(section)
if err != nil {
fmt.Printf("[ERROR] %s() from %s %s.\n", loginfo, file, err)
os.Exit(retFailed)
}
keystrs := items.KeyStrings()
if len(keystrs) <= 0 {
fmt.Printf("[ERROR] %s() get single hostname from %s failed.", loginfo, file)
os.Exit(retFailed)
}
for _, v := range keystrs {
v = strings.TrimSpace(v)
if v != "" {
tokens := strings.Split(v, " ")
if len(tokens) == 0 {
continue
}
host := tokens[0]
if strings.HasPrefix(host, "(") && strings.HasSuffix(host, ")") {
srp := strings.NewReplacer("(", "", ")", "")
host = srp.Replace(host)
} else if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") {
srp := strings.NewReplacer("[", "", "]", "")
host = srp.Replace(host)
}
tokens = strings.Split(host, " ")
if len(tokens) == 0 {
continue
}
hostname = tokens[0]
// Three cases to check:
// 0. A hostname that contains a range pesudo-code and a port
// 1. A hostname that contains just a port
if strings.Count(hostname, ":") > 1 {
// Possible an IPv6 address, or maybe a host line with multiple ranges
// IPv6 with Port XXX:XXX::XXX.port
// FQDN foo.example.com
if strings.Count(hostname, ".") == 1 {
hostname = hostname[0:strings.LastIndex(hostname, ".")]
}
} else if (strings.Count(hostname, "[") > 0 && strings.Count(hostname, "]") > 0 &&
(strings.LastIndex(hostname, "]") < strings.LastIndex(hostname, ":"))) ||
((strings.Count(hostname, "]") <= 0) && (strings.Count(hostname, ":") > 0)) {
hostname = hostname[0:strings.LastIndex(hostname, ":")]
}
}
if hostname != "" {
break
}
}
return hostname
}
func main() {
var ini_cfg *ini.File
var err error
if _, err = isExists(ANSIBLE_CMD); err != nil {
ANSIBLE_CMD = "ansible-playbook"
}
if *version {
fmt.Printf("%s: %s\n", os.Args[0], VERSION)
os.Exit(retOk)
}
if *operation_file == "" || *inventory_file == "" {
fmt.Printf("[ERROR] Not supported action: %s\n", action)
//gLogger.Error("operation and inventory file must provide.\n")
flag.Usage()
os.Exit(retFailed)
} else {
ret := checkExistFiles(*operation_file, *inventory_file)
if !ret {
fmt.Printf("[ERROR] check exists of operation and inventory file.\n")
//gLogger.Error("check exists of operation and inventory file.\n")
os.Exit(retInvaidArgs)
}
}
if action == "" {
fmt.Printf("[ERROR] action(check,update,deploy,rollback) must provide one.\n\n")
//gLogger.Error("action(check,update,deploy,rollback) must provide one.\n")
flag.Usage()
os.Exit(retInvaidArgs)
}
if *retry_file != "" {
if *retry_file, err = filepath.Abs(*retry_file); err != nil {
panic(fmt.Errorf("get Abs path of %s failed: %s\n", *retry_file, err))
//gLogger.Error("get Abs path of %s failed: %s\n", *retry_file, err)
os.Exit(retInvaidArgs)
}
}
if *inventory_file, err = filepath.Abs(*inventory_file); err != nil {
panic(fmt.Errorf("get Abs path of %s failed: %s\n", *inventory_file, err))
} else {
ini_cfg, err = ini.Load(*inventory_file)
if err != nil {
panic(fmt.Errorf("ini load conf failed: %s\n", err))
}
}
if *operation_file, err = filepath.Abs(*operation_file); err != nil {
panic(fmt.Errorf("get Abs path of %s failed: %s\n", *operation_file, err))
}
fmt.Printf("[%s] action on [%s]\n", action, *operation_file)
//gLogger.Info("[%s] action on [%s]\n", action, *operation_file)
switch action {
case "check":
fmt.Printf("-------------Now doing in action: %s\n", action)
//gLogger.Info("-------------Now doing in action: %s\n", action)
fmt.Printf("inventory: %s\n", *inventory_file)
ini_cfg.WriteTo(os.Stdout)
fmt.Println("")
opYamlSyntaxCheck(action, *inventory_file, *operation_file, *extra_vars, "all")
displayYamlFile(*operation_file)
case "update":
fmt.Printf("-------------Now doing in action: %s\n", action)
//gLogger.Info("-------------Now doing in action: %s\n", action)
doUpdateAction(action, *inventory_file, *operation_file, *program_version, *concurrent)
case "deploy":
fmt.Printf("-------------Inventory file is: %s\n", *inventory_file)
//gLogger.Info("-------------Inventory file is: %s\n", *inventory_file)
fmt.Printf("inventory: %s\n", *inventory_file)
ini_cfg.WriteTo(os.Stdout)
fmt.Printf("-------------Now doing in action:[%s], single mode:[%t]\n", action, *single_mode)
doDeployAction(action, *inventory_file, *operation_file, *single_mode, *concurrent, *retry_file, *extra_vars, *section)
case "rollback":
fmt.Printf("-------------Now doing in action: %s\n", action)
//gLogger.Info("-------------Now doing in action: %s\n", action)
fmt.Println("rollback action do nothing now.")
default:
fmt.Printf("Not supported action: %s\n", action)
//gLogger.Info("Not supported action: %s\n", action)
os.Exit(retFailed)
}
}
