func fsGetData(log *logging.Logger, section *common.Section, cfg *ini.File) {
var previous int
var res common.JSONFile
var db database.DB
if section.Dataset-1 == 0 {
previous = cfg.Section("dataset").Key(section.Grace).MustInt()
} else {
previous = section.Dataset - 1
}
db.Open(log, cfg)
defer db.Close()
for _, item := range []string{"directory", "file", "symlink"} {
for res = range database.ListItems(log, &db, section, item) {
switch item {
case "directory":
fsSaveData(log, cfg, section, res, false)
case "symlink":
fsSaveData(log, cfg, section, res, false)
case "file":
if database.ItemExist(log, &db, &res, section, previous) {
fsSaveData(log, cfg, section, res, true)
} else {
fsSaveData(log, cfg, section, res, false)
}
}
}
}
}
func Serve(log *logging.Logger, addr string, ssl *ini.File) {
var cmd []byte
var ln net.Listener
if ssl != nil {
certificate := ssl.Section("ssl").Key("certificate").String()
key := ssl.Section("ssl").Key("key").String()
ln = tlsServe(log, addr, certificate, key)
log.Debug("Opened SSL socket")
} else {
ln = plainServe(log, addr)
log.Debug("Opened plain socket")
}
defer ln.Close()
for {
conn, err := ln.Accept()
buff := bufio.NewReader(conn)
if err != nil {
// handle error
log.Error("Error: %v\n", err)
}
log.Debug("Connection from " + conn.RemoteAddr().String() + " accepted")
cmd, err = buff.ReadBytes('\n')
log.Debug("Remote data readed")
Parse(log, cmd, conn)
conn.Close()
}
}
func fileBackup(log *logging.Logger, section *common.Section, cfg *ini.File, c chan bool, wg *sync.WaitGroup) {
defer func() {
<-c
wg.Done()
}()
dataset.DelDataset(log, cfg, section.Name, section.Grace, section.Dataset)
log.Info("About to backup section " + section.Name)
// Execute pre_command
if err := execCommand(log, cfg.Section(section.Name), "pre_command"); err != nil {
log.Debug(err.Error())
// TODO: manage error
}
// Backup!
backup.Filebackup(log, section, cfg)
// Execute post_command
if err := execCommand(log, cfg.Section(section.Name), "post_command"); err != nil {
log.Debug(err.Error())
// TODO: manage error
}
}
func fileRestore(log *logging.Logger, cfg *ini.File, section *common.Section) {
// Execute pre_command
execCommand(log, cfg.Section(section.Name), "pre_command")
// Restore!
restore.FileRestore(log, section, cfg)
// Execute post_command
execCommand(log, cfg.Section(section.Name), "post_command")
}
func Backup(log *logging.Logger, cfg *ini.File, grace string, reload bool) {
const POOL = 5
var db database.DB
var tx *sql.Tx
var c = make(chan bool, POOL)
var wg = new(sync.WaitGroup)
var dataset, maxdatasets int
var sections []*ini.Section
sections = cfg.Sections()
maxdatasets, _ = cfg.Section("dataset").Key(grace).Int()
db.Open(log, cfg)
defer db.Close()
tx, _ = db.Conn.Begin()
dataset = database.GetDataset(log, tx, grace)
tx.Commit()
if !reload {
if nextds := dataset + 1; nextds > maxdatasets {
dataset = 1
} else {
dataset = dataset + 1
}
}
log.Info("Dataset processed: " + strconv.Itoa(dataset))
wg.Add(len(sections) - len(SECT_RESERVED))
for _, section := range sections {
if !contains(SECT_RESERVED, section.Name()) {
if section.Key("type").String() == "file" { // FIXME: useless?
sect := common.Section{
Name: section.Name(),
Grace: grace,
Dataset: dataset,
Compressed: section.Key("compress").MustBool(),
}
go fileBackup(log, §, cfg, c, wg)
c <- true
}
}
}
wg.Wait() // Wait for all the children to die
close(c)
tx, _ = db.Conn.Begin()
database.SetDataset(log, tx, dataset, grace)
tx.Commit()
}
func (db *DB) retrieveCfgData(cfg *ini.File) {
db.User = cfg.Section("database").Key("user").String()
db.Password = cfg.Section("database").Key("password").String()
db.Host = cfg.Section("database").Key("host").String()
db.Port = cfg.Section("database").Key("port").String()
db.Database = cfg.Section("database").Key("dbname").String()
}
func Restore(log *logging.Logger, cfg *ini.File, grace string) {
dataset := cfg.Section("general").Key("dataset").MustInt()
for _, section := range cfg.Sections() {
if !contains(SECT_RESERVED, section.Name()) {
if section.Key("type").String() == "file" {
sect := common.Section{
Name: section.Name(),
Grace: grace,
Dataset: dataset,
Compressed: section.Key("compress").MustBool(),
}
fileRestore(log, cfg, §)
}
}
}
}
func FileRestore(log *logging.Logger, section *common.Section, cfg *ini.File) {
var cmd common.JSONMessage
var res common.JSONFile
var db database.DB
var err error
db.Open(log, cfg)
defer db.Close()
cmd.Context = "file"
cmd.Command.Name = "put"
if cfg.Section(section.Name).Key("path").String() == "" {
log.Debug("About to full restore")
cmd.Command.ACL = cfg.Section(section.Name).Key("acl").MustBool()
for _, item := range []string{"directory", "file", "symlink"} {
for res = range database.ListItems(log, &db, section, item) {
if cmd.Command.ACL {
log.Debug("About to restore ACLs for " + res.Name)
res.Acl, err = database.GetAcls(log, &db, res.Name, section)
if err != nil {
log.Error("ACLs extraction error for " + res.Name)
log.Debug("error: " + err.Error())
}
}
cmd.Command.Element = res
put(log, section, cfg, &cmd)
}
}
} else {
log.Debug("About to restore path " + cfg.Section(section.Name).Key("path").String())
cmd.Command.Element, err = database.GetItem(log, &db, cfg.Section(section.Name).Key("path").String(), section)
if err != nil {
log.Error("Error when putting data to section " + section.Name)
log.Debug("error: " + err.Error())
} else {
cmd.Command.ACL = cfg.Section(section.Name).Key("acl").MustBool()
put(log, section, cfg, &cmd)
}
}
}
func put(log *logging.Logger, section *common.Section, cfg *ini.File, cmd *common.JSONMessage) {
var conn net.Conn
var err error
conn, err = network.GetSocket(cfg.Section(section.Name))
if err != nil {
log.Error("Error when opening connection with section " + section.Name)
log.Debug("error: " + err.Error())
return
}
defer conn.Close()
cmd.Send(conn)
if cmd.Command.Element.Type == "file" {
transfered := dataset.Path(cfg, section, false) + string(filepath.Separator) + cmd.Command.Element.Name
if err := common.SendFile(transfered, conn); err != nil {
log.Debug("Error when sending file: ", err.Error())
}
}
}
func Path(cfg *ini.File, section *common.Section, previous bool) string {
var dataset int
if previous {
if section.Dataset == 1 {
dataset = cfg.Section("dataset").Key(section.Grace).MustInt()
} else {
dataset = section.Dataset - 1
}
} else {
dataset = section.Dataset
}
destination := strings.Join([]string{
cfg.Section("general").Key("repository").String(),
section.Grace,
strconv.Itoa(dataset),
section.Name,
}, string(filepath.Separator))
return destination
}
func main() {
var cfg *ini.File
var backup bool
var reload bool
var err error
s := options.NewOptions(SPEC)
// Check if options isn't passed
if len(os.Args[1:]) <= 0 {
s.PrintUsageAndExit("No option specified")
}
opts := s.Parse(os.Args[1:])
grace := ""
// Print version and exit
if opts.GetBool("version") {
fmt.Println("Memento server " + VERSION)
os.Exit(0)
}
// Print help and exit
if opts.GetBool("help") {
s.PrintUsageAndExit("Memento server " + VERSION)
}
// Check backup or restore operation (mandatory)
if opts.GetBool("backup") && opts.GetBool("restore") {
// If backup and restore options are passed in the same session, print help and exit
s.PrintUsageAndExit("Cannot perform a backup and restore operations in the same session")
}
// Read grace (mandatory)
if opts.GetBool("hour") {
grace = "hour"
} else if opts.GetBool("day") {
grace = "day"
} else if opts.GetBool("week") {
grace = "week"
} else if opts.GetBool("month") {
grace = "month"
} else {
// If grace is not selected, print help and exit
s.PrintUsageAndExit("No grace selected")
}
if opts.GetBool("reload-dataset") {
reload = true
} else {
reload = false
}
if opts.GetBool("backup") {
backup = true
cfg, err = ini.Load([]byte{}, opts.Get("backup"))
} else if opts.GetBool("restore") {
backup = false
cfg, err = ini.Load([]byte{}, opts.Get("restore"))
}
if err != nil {
fmt.Println("Error about reading config file:", err)
os.Exit(1)
}
repository := cfg.Section("general").Key("repository").String()
checkStructure(repository)
loglevel, _ := logging.LogLevel(cfg.Section("general").Key("log_level").String())
log := setLog(loglevel, cfg.Section("general").Key("log_file").String())
log.Info("Started version " + VERSION)
log.Debug("Grace selected: " + grace)
if backup {
server.Backup(log, cfg, grace, reload)
} else {
server.Restore(log, cfg, grace)
}
log.Info("Ended version " + VERSION)
}
func fsSaveData(log *logging.Logger, cfg *ini.File, section *common.Section, data common.JSONFile, previous bool) {
var item, source, dest, hash string
var cmd common.JSONMessage
var conn net.Conn
var err error
item = dataset.ConvertPath(data.Os, data.Name)
source = dataset.Path(cfg, section, true) + string(filepath.Separator) + item
dest = dataset.Path(cfg, section, false) + string(filepath.Separator) + item
log.Debug("Save item: " + dest)
switch data.Type {
case "directory":
os.MkdirAll(dest, 0755)
case "symlink":
if err := os.Symlink(data.Link, dest); err != nil {
log.Error("Error when creating symlink for file %s", data.Name)
log.Debug("Trace: %s", err.Error())
}
case "file":
if previous {
if section.Compressed {
source = source + ".compressed"
dest = dest + ".compressed"
}
if err = os.Link(source, dest); err != nil {
log.Error("Error when link file %s", data.Name)
log.Debug("Trace: " + err.Error())
}
} else {
cmd.Context = "file"
cmd.Command.Name = "get"
cmd.Command.Element = data
conn, err = network.GetSocket(cfg.Section(section.Name))
if err != nil {
log.Error("Error when opening connection with section " + section.Name)
log.Debug("Trace: " + err.Error())
return
}
defer conn.Close()
cmd.Send(conn)
if hash, err = common.ReceiveFile(dest, conn); err != nil {
log.Error("Error when receiving file " + data.Name)
log.Debug("Trace: " + err.Error())
}
// TODO: check file's hash
if hash == "" {
log.Error("Hash for file " + dest + " mismatch")
// TODO: remove file if hash mismatch
} else {
log.Debug("Hash for file " + dest + " is " + hash)
if section.Compressed {
dataset.CompressFile(log, dest)
}
}
}
}
}
// Receives a list of Container INI configuration files and load them into
// "Container" section of primary Config instance.
func (cfg *Config) loadIniConfigs(cfgPaths []string) error {
var err error
var iniCfg *ini.File
var section *ini.Section
var sectionName string
var cfgPath string
var serviceCfg *ServiceConfig
var containerCfg *ContainerConfig
var name string
var match bool
for _, cfgPath = range cfgPaths {
log.Printf("[Config] Loading: '%s'", cfgPath)
// avoiding dummy files
if match, _ = path.Match("\\.\\#*\\.ini", path.Base(cfgPath)); match {
log.Printf("[Config] Ignoring config file: '%s'", cfgPath)
continue
}
if iniCfg, err = ini.Load(cfgPath); err != nil {
log.Println("[Config] Config load error:", err)
return err
}
// the section names will determine witch kind of configuratio this is
for _, sectionName = range iniCfg.SectionStrings() {
section = iniCfg.Section(sectionName)
switch sectionName {
case "Service":
serviceCfg = new(ServiceConfig)
if err = section.MapTo(serviceCfg); err != nil {
log.Println("[Config] Error on mapTo ServiceConfig:", err)
return err
}
if name, err = sanitizeName(serviceCfg.Name); err != nil {
log.Println("[Config] Error on sanitize name:", err)
return err
}
log.Printf("[Config] Adding service: '%s'", name)
serviceCfg.Name = name
cfg.Service[name] = serviceCfg
log.Printf("[Config] DEBUG serviceCfg: '%+v'", serviceCfg)
case "Container":
containerCfg = new(ContainerConfig)
if err = section.MapTo(containerCfg); err != nil {
log.Println("[Config] Error on mapTo Container:", err)
return err
}
if name, err = sanitizeName(containerCfg.Name); err != nil {
log.Println("[Config] Error on sanitize name:", err)
return err
}
log.Printf("[Config] Adding container: '%s'", name)
containerCfg.Name = name
cfg.Container[name] = containerCfg
log.Printf("[Config] DEBUG containerCfg: '%+v'", containerCfg)
case "DEFAULT":
continue
default:
log.Printf("[Config] Ignored section: '%s'", sectionName)
}
}
}
return nil
}
func fsGetMetadata(log *logging.Logger, section *common.Section, cfg *ini.File) {
var conn net.Conn
var cmd common.JSONMessage
var db database.DB
var buff *bufio.Reader
var res common.JSONResult
var tx *sql.Tx
var data []byte
var err error
log.Debug("Getting metadata for " + section.Name)
cmd.Context = "file"
cmd.Command.Name = "list"
cmd.Command.Paths = strings.Split(cfg.Section(section.Name).Key("path").String(), ",")
cmd.Command.ACL = cfg.Section(section.Name).Key("acl").MustBool()
cmd.Command.Exclude = cfg.Section(section.Name).Key("exclude").String()
log.Debug("Metadata command request: %+v", cmd)
conn, err = network.GetSocket(cfg.Section(section.Name))
if err != nil {
log.Error("Error when opening connection with section " + section.Name)
log.Debug("error: " + err.Error())
return
}
defer conn.Close()
cmd.Send(conn)
db.Open(log, cfg)
defer db.Close()
tx, err = db.Conn.Begin()
if err != nil {
log.Error("Transaction error for section " + section.Name)
log.Debug("Trace: " + err.Error())
return
}
buff = bufio.NewReader(conn)
for {
res = common.JSONResult{}
data, err = buff.ReadBytes('\n')
if err != nil {
if err == io.EOF {
log.Debug("All files's metadata are saved")
break
}
log.Error("Error when getting files metadata for section " + section.Name)
log.Debug("error: " + err.Error())
return
}
err = json.Unmarshal(data, &res)
if err != nil {
log.Error("Error when parsing JSON data for section " + section.Name)
log.Debug("error: " + err.Error())
return
}
log.Debug("Metadata received: %+v", res)
if err = database.SaveAttrs(log, tx, section, res.Data); err != nil {
log.Error("Failed saving database item: " + res.Data.Name)
log.Debug("Trace: " + err.Error())
tx.Rollback()
return
}
if err = database.SaveAcls(log, tx, section, res.Data.Name, res.Data.Acl); err != nil {
log.Error("Failed saving ACLs into database for item: " + res.Data.Name)
log.Debug("Trace: " + err.Error())
tx.Rollback()
return
}
}
tx.Commit()
}