func Restore(snapshotName string) {
	snapshot := common.LoadSnapshot(common.Cfg.BackupSet.SnapshotsDir + snapshotName)

	log.Info("Restoring bucket %s to snapshot %s.", common.Cfg.BackupSet.MasterBucket, snapshotName)

	common.ConfigureAws(common.Cfg.BackupSet.MasterRegion)

	for i := 0; i < common.RestoreWorkerCount; i++ {
		readyRestoreWorkers <- i
		go downloadWorker()
		go uploadWorker()
	}

	for _, version := range snapshot.Contents {
		wid := <-readyRestoreWorkers
		downloadWorkQueue <- DownloadWork{Wid: wid, Version: version, Retry: 0}
	}

	for i := common.RestoreWorkerCount; i > 0; i-- {
		log.Info("Wait for %d restore workers to finish.", i)
		wid := <-readyRestoreWorkers
		log.Info("Restore worker [%d] finished.", wid)
	}
	close(downloadWorkQueue)
	close(uploadWorkQueue)

	log.Info("Restored bucket %s to snapshot %s.", common.Cfg.BackupSet.MasterBucket, snapshotName)
}
func Snapshot() {

	timestamp := time.Now()
	timestampStr := timestamp.Format("20060102150405-0700MST")
	log.Info("Taking snapshot %s of bucket %s.", timestampStr, common.Cfg.BackupSet.SlaveBucket)

	common.ConfigureAws(common.Cfg.BackupSet.SlaveRegion)
	for wid := 0; wid < common.SnapshotWorkerCount; wid++ {
		readySnapshotWorkers <- wid
	}

	go func() { workRequests <- "" }()
	go dispatchWorkers()

	for newVersions := range versionsFunnel {
		versions = append(versions, newVersions...)
	}

	log.Info("Dumping snapshot to %s%s.", common.Cfg.BackupSet.SnapshotsDir, timestampStr)
	snapshot := &common.Snapshot{
		File:      common.Cfg.BackupSet.SnapshotsDir + "/" + timestampStr,
		Timestamp: timestamp,
		Contents:  versions,
	}
	if common.Cfg.BackupSet.CompressSnapshots {
		snapshot.File += ".Z"
	}
	bytes, err := json.MarshalIndent(snapshot, "", "    ")
	if err != nil {
		log.Fatal("Could not marshal snapshot %s: %s", timestampStr, err)
	}
	if common.Cfg.BackupSet.CompressSnapshots {
		f, openErr := os.OpenFile(snapshot.File, os.O_WRONLY|os.O_CREATE, 0644)
		if openErr != nil {
			log.Fatal("Could not open file %s: %s", snapshot.File, openErr)
		}
		defer f.Close()

		w := gzip.NewWriter(f)
		if _, writeErr := w.Write(bytes); writeErr != nil {
			log.Fatal("Could not write compressed snapshot file %s: %s", snapshot.File, writeErr)
		}
		w.Close()
	} else {
		if writeErr := ioutil.WriteFile(snapshot.File, bytes, 0644); err != nil {
			log.Fatal("Could not write snapshot file %s: %s", snapshot.File, writeErr)
		}
	}
	log.Info("Snapshot %s of bucket %s is DONE.", timestampStr, common.Cfg.BackupSet.SlaveBucket)
}
Exemple #3
0
func removeVersions(versionsToRemove []common.Version) (ok bool) {
	objectBatches := makeObjectBatches(versionsToRemove)

	common.ConfigureAws(common.Cfg.BackupSet.SlaveRegion)

	s3Client := s3.New(nil)

	for batch, objects := range objectBatches {
		params := &s3.DeleteObjectsInput{
			Bucket: aws.String(common.Cfg.BackupSet.SlaveBucket),
			Delete: &s3.Delete{
				Objects: objects,
				Quiet:   aws.Bool(true),
			},
		}

		log.Debug("params[%d] = %+v", batch, *params)
		resp, err := s3Client.DeleteObjects(params)

		if err != nil {
			awsErr, ok := err.(awserr.Error)
			if !ok {
				// According to aws-sdk-go documentation:
				// This case should never be hit, The SDK should alwsy return an
				// error which satisfies the awserr.Error interface.
				log.Error(err.Error())
				return false
			}

			// Generic AWS Error with Code, Message, and original error (if any)
			log.Error(awsErr.Code(), awsErr.Message(), awsErr.OrigErr())
			if reqErr, ok := err.(awserr.RequestFailure); ok {
				// A service error occurred
				log.Error(reqErr.Code(), reqErr.Message(), reqErr.StatusCode(), reqErr.RequestID())
			}

			if awsErr.Code() != "NoSuchVersion" {
				return false
			}
		}

		log.Debug("response[%d] = %+v", batch, *resp)
	}
	return true
}