func CreateSingleFileBackup(volume *Volume, snapshot *Snapshot, filePath, destURL string) (string, error) {
driver, err := GetObjectStoreDriver(destURL)
if err != nil {
return "", err
}
if err := addVolume(volume, driver); err != nil {
return "", err
}
volume, err = loadVolume(volume.Name, driver)
if err != nil {
return "", err
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_START,
LOG_FIELD_EVENT: LOG_EVENT_BACKUP,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_SNAPSHOT: snapshot.Name,
LOG_FIELD_FILEPATH: filePath,
}).Debug("Creating backup")
backup := &Backup{
Name: util.GenerateName("backup"),
VolumeName: volume.Name,
SnapshotName: snapshot.Name,
SnapshotCreatedAt: snapshot.CreatedTime,
}
backup.SingleFile.FilePath = getSingleFileBackupFilePath(backup)
if err := driver.Upload(filePath, backup.SingleFile.FilePath); err != nil {
return "", err
}
backup.CreatedTime = util.Now()
if err := saveBackup(backup, driver); err != nil {
return "", err
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_COMPLETE,
LOG_FIELD_EVENT: LOG_EVENT_BACKUP,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_SNAPSHOT: snapshot.Name,
}).Debug("Created backup")
return encodeBackupURL(backup.Name, volume.Name, destURL), nil
}
func (s *daemon) processVolumeCreate(request *api.VolumeCreateRequest) (*Volume, error) {
volumeName := request.Name
driverName := request.DriverName
if volumeName == "" {
volumeName = util.GenerateName("volume")
for s.NameUUIDIndex.Get(volumeName) != "" {
volumeName = util.GenerateName("volume")
}
} else {
if s.NameUUIDIndex.Get(volumeName) != "" {
return nil, fmt.Errorf("Volume %v already exists ", volumeName)
}
}
if driverName == "" {
driverName = s.DefaultDriver
}
driver, err := s.getDriver(driverName)
if err != nil {
return nil, err
}
volOps, err := driver.VolumeOps()
if err != nil {
return nil, err
}
req := Request{
Name: volumeName,
Options: map[string]string{
OPT_SIZE: strconv.FormatInt(request.Size, 10),
OPT_BACKUP_URL: util.UnescapeURL(request.BackupURL),
OPT_VOLUME_NAME: volumeName,
OPT_VOLUME_DRIVER_ID: request.DriverVolumeID,
OPT_VOLUME_TYPE: request.Type,
OPT_VOLUME_IOPS: strconv.FormatInt(request.IOPS, 10),
OPT_PREPARE_FOR_VM: strconv.FormatBool(request.PrepareForVM),
},
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_PREPARE,
LOG_FIELD_EVENT: LOG_EVENT_CREATE,
LOG_FIELD_OBJECT: LOG_OBJECT_VOLUME,
LOG_FIELD_VOLUME: volumeName,
LOG_FIELD_OPTS: req.Options,
}).Debug()
if err := volOps.CreateVolume(req); err != nil {
return nil, err
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_COMPLETE,
LOG_FIELD_EVENT: LOG_EVENT_CREATE,
LOG_FIELD_OBJECT: LOG_OBJECT_VOLUME,
LOG_FIELD_VOLUME: volumeName,
}).Debug("Created volume")
volume := &Volume{
Name: volumeName,
DriverName: driverName,
}
if err := s.NameUUIDIndex.Add(volumeName, "exists"); err != nil {
return nil, err
}
return volume, nil
}
func (s *daemon) doSnapshotCreate(version string, w http.ResponseWriter, r *http.Request, objs map[string]string) error {
request := &api.SnapshotCreateRequest{}
if err := decodeRequest(r, request); err != nil {
return err
}
volumeName := request.VolumeName
if err := util.CheckName(volumeName); err != nil {
return err
}
volume := s.getVolume(volumeName)
if volume == nil {
return fmt.Errorf("volume %v doesn't exist", volumeName)
}
snapshotName := request.Name
if snapshotName != "" {
if err := util.CheckName(snapshotName); err != nil {
return err
}
existName := s.NameUUIDIndex.Get(snapshotName)
if existName != "" {
return fmt.Errorf("Snapshot name %v already exists", snapshotName)
}
} else {
snapshotName = util.GenerateName("snapshot")
for s.NameUUIDIndex.Get(snapshotName) != "" {
snapshotName = util.GenerateName("snapshot")
}
}
snapOps, err := s.getSnapshotOpsForVolume(volume)
if err != nil {
return err
}
req := Request{
Name: snapshotName,
Options: map[string]string{
OPT_VOLUME_NAME: volumeName,
},
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_PREPARE,
LOG_FIELD_EVENT: LOG_EVENT_CREATE,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_SNAPSHOT: snapshotName,
LOG_FIELD_VOLUME: volumeName,
}).Debug()
if err := snapOps.CreateSnapshot(req); err != nil {
return err
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_COMPLETE,
LOG_FIELD_EVENT: LOG_EVENT_CREATE,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_SNAPSHOT: snapshotName,
LOG_FIELD_VOLUME: volumeName,
}).Debug()
//TODO: error handling
if err := s.SnapshotVolumeIndex.Add(snapshotName, volume.Name); err != nil {
return err
}
if err := s.NameUUIDIndex.Add(snapshotName, "exists"); err != nil {
return err
}
driverInfo, err := s.getSnapshotDriverInfo(snapshotName, volume)
if err != nil {
return err
}
if request.Verbose {
return writeResponseOutput(w, api.SnapshotResponse{
Name: snapshotName,
VolumeName: volume.Name,
CreatedTime: driverInfo[OPT_SNAPSHOT_CREATED_TIME],
DriverInfo: driverInfo,
})
}
return writeStringResponse(w, snapshotName)
}
func CreateDeltaBlockBackup(volume *Volume, snapshot *Snapshot, destURL string, sDriver convoydriver.ConvoyDriver) (string, error) {
deltaOps, ok := sDriver.(DeltaBlockBackupOperations)
if !ok {
return "", fmt.Errorf("Driver %s doesn't implemented DeltaBlockBackupOperations interface", sDriver.Name())
}
bsDriver, err := GetObjectStoreDriver(destURL)
if err != nil {
return "", err
}
if err := addVolume(volume, bsDriver); err != nil {
return "", err
}
// Update volume from objectstore
volume, err = loadVolume(volume.Name, bsDriver)
if err != nil {
return "", err
}
lastBackupName := volume.LastBackupName
var lastSnapshotName string
var lastBackup *Backup
if lastBackupName != "" {
lastBackup, err = loadBackup(lastBackupName, volume.Name, bsDriver)
if err != nil {
return "", err
}
lastSnapshotName = lastBackup.SnapshotName
if lastSnapshotName == snapshot.Name {
//Generate full snapshot if the snapshot has been backed up last time
lastSnapshotName = ""
log.Debug("Would create full snapshot metadata")
} else if !deltaOps.HasSnapshot(lastSnapshotName, volume.Name) {
// It's possible that the snapshot in objectstore doesn't exist
// in local storage
lastSnapshotName = ""
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_FALLBACK,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_SNAPSHOT: lastSnapshotName,
LOG_FIELD_VOLUME: volume.Name,
}).Debug("Cannot find last snapshot in local storage, would process with full backup")
}
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_START,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_EVENT: LOG_EVENT_COMPARE,
LOG_FIELD_SNAPSHOT: snapshot.Name,
LOG_FIELD_LAST_SNAPSHOT: lastSnapshotName,
}).Debug("Generating snapshot changed blocks metadata")
delta, err := deltaOps.CompareSnapshot(snapshot.Name, lastSnapshotName, volume.Name)
if err != nil {
return "", err
}
if delta.BlockSize != DEFAULT_BLOCK_SIZE {
return "", fmt.Errorf("Currently doesn't support different block sizes driver other than %v", DEFAULT_BLOCK_SIZE)
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_COMPLETE,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_EVENT: LOG_EVENT_COMPARE,
LOG_FIELD_SNAPSHOT: snapshot.Name,
LOG_FIELD_LAST_SNAPSHOT: lastSnapshotName,
}).Debug("Generated snapshot changed blocks metadata")
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_START,
LOG_FIELD_EVENT: LOG_EVENT_BACKUP,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_SNAPSHOT: snapshot.Name,
}).Debug("Creating backup")
deltaBackup := &Backup{
Name: util.GenerateName("backup"),
VolumeName: volume.Name,
SnapshotName: snapshot.Name,
Blocks: []BlockMapping{},
}
if err := deltaOps.OpenSnapshot(snapshot.Name, volume.Name); err != nil {
return "", err
}
defer deltaOps.CloseSnapshot(snapshot.Name, volume.Name)
mCounts := len(delta.Mappings)
for m, d := range delta.Mappings {
block := make([]byte, DEFAULT_BLOCK_SIZE)
blkCounts := d.Size / delta.BlockSize
for i := int64(0); i < blkCounts; i++ {
offset := d.Offset + i*delta.BlockSize
log.Debugf("Backup for %v: segment %v/%v, blocks %v/%v", snapshot.Name, m+1, mCounts, i+1, blkCounts)
err := deltaOps.ReadSnapshot(snapshot.Name, volume.Name, offset, block)
if err != nil {
return "", err
}
checksum := util.GetChecksum(block)
blkFile := getBlockFilePath(volume.Name, checksum)
if bsDriver.FileSize(blkFile) >= 0 {
blockMapping := BlockMapping{
Offset: offset,
BlockChecksum: checksum,
}
deltaBackup.Blocks = append(deltaBackup.Blocks, blockMapping)
log.Debugf("Found existed block match at %v", blkFile)
continue
}
rs, err := util.CompressData(block)
if err != nil {
return "", err
}
if err := bsDriver.Write(blkFile, rs); err != nil {
return "", err
}
log.Debugf("Created new block file at %v", blkFile)
blockMapping := BlockMapping{
Offset: offset,
BlockChecksum: checksum,
}
deltaBackup.Blocks = append(deltaBackup.Blocks, blockMapping)
}
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_COMPLETE,
LOG_FIELD_EVENT: LOG_EVENT_BACKUP,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_SNAPSHOT: snapshot.Name,
}).Debug("Created snapshot changed blocks")
backup := mergeSnapshotMap(deltaBackup, lastBackup)
backup.SnapshotName = snapshot.Name
backup.SnapshotCreatedAt = snapshot.CreatedTime
backup.CreatedTime = util.Now()
if err := saveBackup(backup, bsDriver); err != nil {
return "", err
}
volume.LastBackupName = backup.Name
if err := saveVolume(volume, bsDriver); err != nil {
return "", err
}
return encodeBackupURL(backup.Name, volume.Name, destURL), nil
}