func Restore(snapshotName string) {
snapshot := common.LoadSnapshot(common.Cfg.BackupSet.SnapshotsDir + snapshotName)
log.Info("Restoring bucket %s to snapshot %s.", common.Cfg.BackupSet.MasterBucket, snapshotName)
common.ConfigureAws(common.Cfg.BackupSet.MasterRegion)
for i := 0; i < common.RestoreWorkerCount; i++ {
readyRestoreWorkers <- i
go downloadWorker()
go uploadWorker()
}
for _, version := range snapshot.Contents {
wid := <-readyRestoreWorkers
downloadWorkQueue <- DownloadWork{Wid: wid, Version: version, Retry: 0}
}
for i := common.RestoreWorkerCount; i > 0; i-- {
log.Info("Wait for %d restore workers to finish.", i)
wid := <-readyRestoreWorkers
log.Info("Restore worker [%d] finished.", wid)
}
close(downloadWorkQueue)
close(uploadWorkQueue)
log.Info("Restored bucket %s to snapshot %s.", common.Cfg.BackupSet.MasterBucket, snapshotName)
}
func Snapshot() {
timestamp := time.Now()
timestampStr := timestamp.Format("20060102150405-0700MST")
log.Info("Taking snapshot %s of bucket %s.", timestampStr, common.Cfg.BackupSet.SlaveBucket)
common.ConfigureAws(common.Cfg.BackupSet.SlaveRegion)
for wid := 0; wid < common.SnapshotWorkerCount; wid++ {
readySnapshotWorkers <- wid
}
go func() { workRequests <- "" }()
go dispatchWorkers()
for newVersions := range versionsFunnel {
versions = append(versions, newVersions...)
}
log.Info("Dumping snapshot to %s%s.", common.Cfg.BackupSet.SnapshotsDir, timestampStr)
snapshot := &common.Snapshot{
File: common.Cfg.BackupSet.SnapshotsDir + "/" + timestampStr,
Timestamp: timestamp,
Contents: versions,
}
if common.Cfg.BackupSet.CompressSnapshots {
snapshot.File += ".Z"
}
bytes, err := json.MarshalIndent(snapshot, "", " ")
if err != nil {
log.Fatal("Could not marshal snapshot %s: %s", timestampStr, err)
}
if common.Cfg.BackupSet.CompressSnapshots {
f, openErr := os.OpenFile(snapshot.File, os.O_WRONLY|os.O_CREATE, 0644)
if openErr != nil {
log.Fatal("Could not open file %s: %s", snapshot.File, openErr)
}
defer f.Close()
w := gzip.NewWriter(f)
if _, writeErr := w.Write(bytes); writeErr != nil {
log.Fatal("Could not write compressed snapshot file %s: %s", snapshot.File, writeErr)
}
w.Close()
} else {
if writeErr := ioutil.WriteFile(snapshot.File, bytes, 0644); err != nil {
log.Fatal("Could not write snapshot file %s: %s", snapshot.File, writeErr)
}
}
log.Info("Snapshot %s of bucket %s is DONE.", timestampStr, common.Cfg.BackupSet.SlaveBucket)
}
func removeVersions(versionsToRemove []common.Version) (ok bool) {
objectBatches := makeObjectBatches(versionsToRemove)
common.ConfigureAws(common.Cfg.BackupSet.SlaveRegion)
s3Client := s3.New(nil)
for batch, objects := range objectBatches {
params := &s3.DeleteObjectsInput{
Bucket: aws.String(common.Cfg.BackupSet.SlaveBucket),
Delete: &s3.Delete{
Objects: objects,
Quiet: aws.Bool(true),
},
}
log.Debug("params[%d] = %+v", batch, *params)
resp, err := s3Client.DeleteObjects(params)
if err != nil {
awsErr, ok := err.(awserr.Error)
if !ok {
// According to aws-sdk-go documentation:
// This case should never be hit, The SDK should alwsy return an
// error which satisfies the awserr.Error interface.
log.Error(err.Error())
return false
}
// Generic AWS Error with Code, Message, and original error (if any)
log.Error(awsErr.Code(), awsErr.Message(), awsErr.OrigErr())
if reqErr, ok := err.(awserr.RequestFailure); ok {
// A service error occurred
log.Error(reqErr.Code(), reqErr.Message(), reqErr.StatusCode(), reqErr.RequestID())
}
if awsErr.Code() != "NoSuchVersion" {
return false
}
}
log.Debug("response[%d] = %+v", batch, *resp)
}
return true
}