func discriminateVersions(oldSnapshots []common.Snapshot, recentSnapshots []common.Snapshot) (versionsToRemove []common.Version) {
recentId := make(map[string]bool)
for _, recentS := range recentSnapshots {
log.Debug("Collecting versions for recent snapshot '%s' on %s.", recentS.File, recentS.Timestamp)
for _, recentVersion := range recentS.Contents {
if common.Cfg.LogLevel >= log.DEBUG {
pretty, _ := json.MarshalIndent(recentVersion, "", " ")
log.Debug("Version is recent: %s", pretty)
}
recentId[recentVersion.VersionId] = true
}
}
for _, oldS := range oldSnapshots {
log.Debug("Discriminating versions for old Snapshot '%s' on %s.", oldS.File, oldS.Timestamp)
for _, oldVersion := range oldS.Contents {
if _, ok := recentId[oldVersion.VersionId]; ok {
continue
}
recentId[oldVersion.VersionId] = false
if common.Cfg.LogLevel > log.DEBUG {
pretty, _ := json.MarshalIndent(oldVersion, "", " ")
log.Debug("Will remove version: %s", pretty)
}
versionsToRemove = append(versionsToRemove, oldVersion)
}
}
return
}
func removeVersions(versionsToRemove []common.Version) (ok bool) {
objectBatches := makeObjectBatches(versionsToRemove)
common.ConfigureAws(common.Cfg.BackupSet.SlaveRegion)
s3Client := s3.New(nil)
for batch, objects := range objectBatches {
params := &s3.DeleteObjectsInput{
Bucket: aws.String(common.Cfg.BackupSet.SlaveBucket),
Delete: &s3.Delete{
Objects: objects,
Quiet: aws.Bool(true),
},
}
log.Debug("params[%d] = %+v", batch, *params)
resp, err := s3Client.DeleteObjects(params)
if err != nil {
awsErr, ok := err.(awserr.Error)
if !ok {
// According to aws-sdk-go documentation:
// This case should never be hit, The SDK should alwsy return an
// error which satisfies the awserr.Error interface.
log.Error(err.Error())
return false
}
// Generic AWS Error with Code, Message, and original error (if any)
log.Error(awsErr.Code(), awsErr.Message(), awsErr.OrigErr())
if reqErr, ok := err.(awserr.RequestFailure); ok {
// A service error occurred
log.Error(reqErr.Code(), reqErr.Message(), reqErr.StatusCode(), reqErr.RequestID())
}
if awsErr.Code() != "NoSuchVersion" {
return false
}
}
log.Debug("response[%d] = %+v", batch, *resp)
}
return true
}
func LoadSnapshot(file string) (snapshot Snapshot) {
log.Info("Loading snapshot file '%s'.", file)
var bytes []byte
var readErr error
if filepath.Ext(file) == ".Z" {
f, openErr := os.OpenFile(file, os.O_RDONLY, 0000)
if openErr != nil {
log.Fatal("Could not open file %s: %s", file, openErr)
}
defer f.Close()
r, nrErr := gzip.NewReader(f)
if nrErr != nil {
log.Fatal("Could not initialize gzip decompressor: %s", nrErr)
}
defer r.Close()
bytes, readErr = ioutil.ReadAll(r)
if readErr != nil {
log.Fatal("Could not read compressed snapshot file %s: %s", file, readErr)
}
} else {
bytes, readErr = ioutil.ReadFile(file)
if readErr != nil {
log.Fatal("Could not read snapshot file '%s': %s", file, readErr)
}
}
err := json.Unmarshal(bytes, &snapshot)
if err != nil {
log.Fatal("Could not parse snapshot file '%s': %s", file, err)
}
snapshot.File = file
if Cfg.LogLevel > 0 {
pretty, _ := json.MarshalIndent(snapshot, "", " ")
log.Debug("Snapshot '%s':\n%s", file, pretty)
}
return
}
func makeObjectBatches(versions []common.Version) (objectBatches [][]*s3.ObjectIdentifier) {
objectBatches = make([][]*s3.ObjectIdentifier, int(math.Ceil(float64(len(versions))/float64(common.GcBatchSize))))
batch := 0
for count := common.Min(len(versions), common.GcBatchSize); count <= len(versions); count += common.GcBatchSize {
objectBatches[batch] = make([]*s3.ObjectIdentifier, common.Min(common.GcBatchSize, count-batch*common.GcBatchSize))
batch++
}
batch = -1
for index, version := range versions {
if index%common.GcBatchSize == 0 {
batch++
}
object := &s3.ObjectIdentifier{
Key: aws.String(version.Key),
VersionId: aws.String(version.VersionId),
}
objectBatches[batch][index-common.GcBatchSize*batch] = object
log.Debug("Batch %d = %+v", batch, objectBatches[batch])
}
return
}
func snapshotWorker(wid int, path string) {
log.Info("[%d] Explore path '%s'.", wid, path)
s3Client := s3.New(nil)
params := &s3.ListObjectVersionsInput{
Bucket: aws.String(common.Cfg.BackupSet.SlaveBucket),
Delimiter: aws.String("/"),
// EncodingType: aws.String("EncodingType"),
// KeyMarker: aws.String("KeyMarker"),
MaxKeys: aws.Int64(common.SnapshotBatchSize),
Prefix: aws.String(path),
// VersionIdMarker: aws.String("VersionIdMarker"),
}
var discoveredVersions []common.Version
buffer := make([]common.Version, common.SnapshotBatchSize)
for batch := 1; ; batch++ {
log.Debug("[%d] Request batch %d for path '%s'", wid, batch, path)
resp, err := s3Client.ListObjectVersions(params)
if err != nil {
if awsErr, ok := err.(awserr.Error); ok {
if reqErr, ok := err.(awserr.RequestFailure); ok {
// A service error occurred
log.Error(awsErr.Code(), awsErr.Message(), awsErr.OrigErr())
log.Fatal(reqErr.Code(), reqErr.Message(), reqErr.StatusCode(), reqErr.RequestID())
} else {
log.Fatal(awsErr.Code(), awsErr.Message(), awsErr.OrigErr())
}
} else {
// This case should never be hit, The SDK should alway return an
// error which satisfies the awserr.Error interface.
log.Fatal(err.Error())
}
}
for _, cp := range resp.CommonPrefixes {
discoveredPath := *cp.Prefix
log.Info("[%d] Discover path '%s'.", wid, discoveredPath)
workRequests <- discoveredPath
}
index := 0
for _, v := range resp.Versions {
if v.IsLatest == nil {
log.Fatal("[%d] IsLatest is nil", wid)
}
if v.IsLatest != nil && *v.IsLatest == true {
if v.Key == nil {
log.Fatal("[%d] Key is nil", wid)
}
if v.LastModified == nil {
log.Fatal("[%d] LastModified is nil", wid)
}
if v.Size == nil {
log.Fatal("[%d] Size is nil", wid)
}
if v.VersionId == nil {
log.Fatal("[%d] VersionId is nil", wid)
}
version := common.Version{
Key: *v.Key,
LastModified: *v.LastModified,
Size: *v.Size,
VersionId: *v.VersionId,
}
log.Debug("[%d] Discover latest version: %s", wid, version)
buffer[index] = version
index++
} else {
log.Debug("[%d] Discover noncurrent latest version for key '%s'.", wid, *v.Key)
}
}
discoveredVersions = append(discoveredVersions, buffer[0:index]...)
if !*resp.IsTruncated {
break
}
log.Info("[%d] Continue exploring path '%s'.", wid, path)
if resp.NextVersionIdMarker != nil {
log.Debug("[%d] NextVersionIdMarker = %+v", wid, resp.NextVersionIdMarker)
} else {
log.Debug("[%d] NextVersionIdMarker = nil", wid)
}
if resp.NextKeyMarker != nil {
log.Debug("[%d] NextKeyMarker = %+v", wid, resp.NextKeyMarker)
} else {
log.Debug("[%d] NextKeyMarker = nil", wid)
}
params.VersionIdMarker = resp.NextVersionIdMarker
params.KeyMarker = resp.NextKeyMarker
}
log.Info("[%d] Registering versions for path '%s'.", wid, path)
versionsFunnel <- discoveredVersions
log.Info("[%d] Done exploring path '%s'.", wid, path)
doneSnapshotWorkers <- wid
}
func downloadWorker() {
s3Client := s3.New(nil)
for work := range downloadWorkQueue {
log.Debug("[%d] Download version, retry %d: %s", work.Wid, work.Retry, work.Version)
getParams := &s3.GetObjectInput{
Bucket: aws.String(common.Cfg.BackupSet.SlaveBucket), // Required
Key: aws.String(work.Version.Key), // Required
// IfMatch: aws.String("IfMatch"),
// IfModifiedSince: aws.Time(time.Now()),
// IfNoneMatch: aws.String("IfNoneMatch"),
// IfUnmodifiedSince: aws.Time(time.Now()),
// Range: aws.String("Range"),
// RequestPayer: aws.String("RequestPayer"),
// ResponseCacheControl: aws.String("ResponseCacheControl"),
// ResponseContentDisposition: aws.String("ResponseContentDisposition"),
// ResponseContentEncoding: aws.String("ResponseContentEncoding"),
// ResponseContentLanguage: aws.String("ResponseContentLanguage"),
// ResponseContentType: aws.String("ResponseContentType"),
// ResponseExpires: aws.Time(time.Now()),
// SSECustomerAlgorithm: aws.String("SSECustomerAlgorithm"),
// SSECustomerKey: aws.String("SSECustomerKey"),
// SSECustomerKeyMD5: aws.String("SSECustomerKeyMD5"),
VersionId: aws.String(work.Version.VersionId),
}
getResp, getErr := s3Client.GetObject(getParams)
if getErr != nil {
if awsErr, ok := getErr.(awserr.Error); ok {
log.Error("[%d] Error code '%s', message '%s', origin '%s'", work.Wid, awsErr.Code(), awsErr.Message(), awsErr.OrigErr())
if reqErr, ok := getErr.(awserr.RequestFailure); ok {
log.Error("[%d] Service error code '%s', message '%s', status code '%d', request id '%s'", work.Wid, reqErr.Code(), reqErr.Message(), reqErr.StatusCode(), reqErr.RequestID())
}
} else {
// This case should never be hit, The SDK should alwsy return an
// error which satisfies the awserr.Error interface.
log.Error("[%d] Non AWS error: %s", work.Wid, getErr.Error())
}
work.Retry++
if work.Retry == common.MaxRetries {
log.Fatal("[%d] Error downloading version, retry %d: %s", work.Wid, work.Retry, work.Version)
}
log.Error("[%d] Error downloading version, retry %d: %s", work.Wid, work.Retry, work.Version)
downloadWorkQueue <- work
continue
}
log.Debug("[%d] Read response: %s", work.Wid, work.Version)
bytes, readErr := ioutil.ReadAll(getResp.Body)
getResp.Body.Close()
if readErr != nil {
work.Retry++
if work.Retry == common.MaxRetries {
log.Fatal("[%d] Could not read version %s, retry %d: %s", work.Wid, work.Version, work.Retry, readErr)
}
log.Error("[%d] Could not read version %s, retry %d: %s", work.Wid, work.Version, work.Retry, readErr)
downloadWorkQueue <- work
continue
}
log.Debug("[%d] Downloaded version: %s", work.Wid, work.Version)
uploadWorkQueue <- UploadWork{Wid: work.Wid, Version: work.Version, Bytes: bytes, Retry: 0}
}
}
func uploadWorker() {
s3Client := s3.New(nil)
for work := range uploadWorkQueue {
log.Debug("[%d] Upload version, retry %d: %s", work.Wid, work.Retry, work.Version)
putParams := &s3.PutObjectInput{
Bucket: aws.String(common.Cfg.BackupSet.MasterBucket), // Required
Key: aws.String(work.Version.Key), // Required
// ACL: aws.String("ObjectCannedACL"),
Body: bytes.NewReader(work.Bytes),
// CacheControl: aws.String("CacheControl"),
// ContentDisposition: aws.String("ContentDisposition"),
// ContentEncoding: aws.String("ContentEncoding"),
// ContentLanguage: aws.String("ContentLanguage"),
// ContentLength: aws.Long(1),
// ContentType: aws.String("ContentType"),
// Expires: aws.Time(time.Now()),
// GrantFullControl: aws.String("GrantFullControl"),
// GrantRead: aws.String("GrantRead"),
// GrantReadACP: aws.String("GrantReadACP"),
// GrantWriteACP: aws.String("GrantWriteACP"),
// Metadata: map[string]*string{
// "Key": aws.String("MetadataValue"), // Required
// // More values...
// },
// RequestPayer: aws.String("RequestPayer"),
// SSECustomerAlgorithm: aws.String("SSECustomerAlgorithm"),
// SSECustomerKey: aws.String("SSECustomerKey"),
// SSECustomerKeyMD5: aws.String("SSECustomerKeyMD5"),
// SSEKMSKeyID: aws.String("SSEKMSKeyId"),
// ServerSideEncryption: aws.String("ServerSideEncryption"),
// StorageClass: aws.String("StorageClass"),
// WebsiteRedirectLocation: aws.String("WebsiteRedirectLocation"),
}
_, putErr := s3Client.PutObject(putParams)
if putErr != nil {
if awsErr, ok := putErr.(awserr.Error); ok {
log.Error("[%d] Error code '%s', message '%s', origin '%s'", work.Wid, awsErr.Code(), awsErr.Message(), awsErr.OrigErr())
if reqErr, ok := putErr.(awserr.RequestFailure); ok {
log.Error("[%d] Service error code '%s', message '%s', status code '%d', request id '%s'", work.Wid, reqErr.Code(), reqErr.Message(), reqErr.StatusCode(), reqErr.RequestID())
}
} else {
// This case should never be hit, The SDK should alwsy return an
// error which satisfies the awserr.Error interface.
log.Error("[%d] Non AWS error: %s", work.Wid, putErr.Error())
}
work.Retry++
if work.Retry == common.MaxRetries {
log.Fatal("[%d] Error uploading version, retry %d: %s", work.Wid, work.Retry, work.Version)
}
log.Error("[%d] Error uploading version, retry %d: %s", work.Wid, work.Retry, work.Version)
uploadWorkQueue <- work
continue
}
log.Info("[%d] Restored version: %s", work.Wid, work.Version)
readyRestoreWorkers <- work.Wid
}
}