func (d *Driver) CreateVolume(req Request) error {
id := req.Name
opts := req.Options
size, err := util.ParseSize(opts[OPT_SIZE])
if err != nil {
return err
}
if size == 0 {
size = d.DefaultVolumeSize
}
volume := d.blankVolume(id)
volume.Size = size
volume.Name = opts[OPT_VOLUME_NAME]
volume.PrepareForVM, err = strconv.ParseBool(opts[OPT_PREPARE_FOR_VM])
volume.CreatedTime = util.Now()
if err != nil {
return err
}
stack := volume.Stack(d)
if err := d.doCreateVolume(volume, stack, id, opts); err != nil {
stack.Delete()
return err
}
return nil
}
func (d *Driver) CreateSnapshot(req Request) error {
d.mutex.Lock()
defer d.mutex.Unlock()
id := req.Name
volumeID, err := util.GetFieldFromOpts(OPT_VOLUME_NAME, req.Options)
if err != nil {
return err
}
volume := d.blankVolume(volumeID)
if err := util.ObjectLoad(volume); err != nil {
return err
}
if _, exists := volume.Snapshots[id]; exists {
return fmt.Errorf("Snapshot %v already exists for volume %v", id, volumeID)
}
snapFile := d.getSnapshotFilePath(id, volumeID)
if err := util.MkdirIfNotExists(filepath.Dir(snapFile)); err != nil {
return err
}
if err := util.CompressDir(volume.Path, snapFile); err != nil {
return err
}
volume.Snapshots[id] = Snapshot{
Name: id,
CreatedTime: util.Now(),
VolumeUUID: volumeID,
FilePath: snapFile,
}
return util.ObjectSave(volume)
}
func (d *Driver) CreateVolume(req Request) error {
d.mutex.Lock()
defer d.mutex.Unlock()
id := req.Name
opts := req.Options
backupURL := opts[OPT_BACKUP_URL]
if backupURL != "" {
objVolume, err := objectstore.LoadVolume(backupURL)
if err != nil {
return err
}
if objVolume.Driver != d.Name() {
return fmt.Errorf("Cannot restore backup of %v to %v", objVolume.Driver, d.Name())
}
}
volume := d.blankVolume(id)
exists, err := util.ObjectExists(volume)
if err != nil {
return err
}
if exists {
return fmt.Errorf("volume %v already exists", id)
}
volume.PrepareForVM, err = strconv.ParseBool(opts[OPT_PREPARE_FOR_VM])
if err != nil {
return err
}
if volume.PrepareForVM {
volume.Size, err = d.getSize(opts, d.DefaultVolumeSize)
if err != nil {
return err
}
}
volumePath := filepath.Join(d.Path, id)
if err := util.MkdirIfNotExists(volumePath); err != nil {
return err
}
volume.Path = volumePath
volume.CreatedTime = util.Now()
volume.Snapshots = make(map[string]Snapshot)
volume.Name = id
if backupURL != "" {
file, err := objectstore.RestoreSingleFileBackup(backupURL, volumePath)
if err != nil {
return err
}
// file would be removed after this because it's under volumePath
if err := util.DecompressDir(file, volumePath); err != nil {
return err
}
}
return util.ObjectSave(volume)
}
func (d *Driver) CreateSnapshot(req Request) error {
var err error
d.mutex.Lock()
defer d.mutex.Unlock()
id := req.Name
volumeID, err := util.GetFieldFromOpts(OPT_VOLUME_NAME, req.Options)
if err != nil {
return err
}
volume := d.blankVolume(volumeID)
if err := util.ObjectLoad(volume); err != nil {
return err
}
devID, err := d.allocateDevID()
if err != nil {
return err
}
snapshot, exists := volume.Snapshots[id]
if exists {
return generateError(logrus.Fields{
LOG_FIELD_VOLUME: volumeID,
LOG_FIELD_SNAPSHOT: id,
}, "Already has snapshot with name")
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_START,
LOG_FIELD_EVENT: LOG_EVENT_CREATE,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_SNAPSHOT: id,
LOG_FIELD_VOLUME: volumeID,
DM_LOG_FIELD_VOLUME_DEVID: volume.DevID,
DM_LOG_FIELD_SNAPSHOT_DEVID: devID,
}).Debugf("Creating snapshot")
err = devicemapper.CreateSnapDevice(d.ThinpoolDevice, devID, volumeID, volume.DevID)
if err != nil {
return err
}
log.Debugf("Created snapshot device")
snapshot = Snapshot{
Name: id,
CreatedTime: util.Now(),
DevID: devID,
Activated: false,
}
volume.Snapshots[id] = snapshot
if err := util.ObjectSave(volume); err != nil {
return err
}
return nil
}
func CreateSingleFileBackup(volume *Volume, snapshot *Snapshot, filePath, destURL string) (string, error) {
driver, err := GetObjectStoreDriver(destURL)
if err != nil {
return "", err
}
if err := addVolume(volume, driver); err != nil {
return "", err
}
volume, err = loadVolume(volume.UUID, driver)
if err != nil {
return "", err
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_START,
LOG_FIELD_EVENT: LOG_EVENT_BACKUP,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_SNAPSHOT: snapshot.UUID,
LOG_FIELD_FILEPATH: filePath,
}).Debug("Creating backup")
backup := &Backup{
UUID: uuid.New(),
VolumeUUID: volume.UUID,
SnapshotUUID: snapshot.UUID,
SnapshotName: snapshot.Name,
SnapshotCreatedAt: snapshot.CreatedTime,
}
backup.SingleFile.FilePath = getSingleFileBackupFilePath(backup)
if err := driver.Upload(filePath, backup.SingleFile.FilePath); err != nil {
return "", err
}
backup.CreatedTime = util.Now()
if err := saveBackup(backup, driver); err != nil {
return "", err
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_COMPLETE,
LOG_FIELD_EVENT: LOG_EVENT_BACKUP,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_SNAPSHOT: snapshot.UUID,
}).Debug("Created backup")
return encodeBackupURL(backup.UUID, volume.UUID, destURL), nil
}
func (d *Driver) CreateVolume(req Request) error {
d.mutex.Lock()
defer d.mutex.Unlock()
id := req.Name
opts := req.Options
volume := d.blankVolume(id)
exists, err := util.ObjectExists(volume)
if err != nil {
return err
}
if exists {
return fmt.Errorf("volume %v already exists", id)
}
volume.PrepareForVM, err = strconv.ParseBool(opts[OPT_PREPARE_FOR_VM])
if err != nil {
return err
}
if volume.PrepareForVM {
volume.Size, err = d.getSize(opts, d.DefaultVolumeSize)
if err != nil {
return err
}
}
gVolume := d.gVolumes[d.DefaultVolumePool]
volumePath := filepath.Join(gVolume.MountPoint, id)
if util.VolumeMountPointFileExists(gVolume, id, util.FILE_TYPE_DIRECTORY) {
log.Debugf("Found existing volume named %v, reuse it", id)
} else if err := util.VolumeMountPointDirectoryCreate(gVolume, id); err != nil {
return err
}
volume.Name = id
volume.Path = volumePath
volume.VolumePool = gVolume.UUID
volume.CreatedTime = util.Now()
return util.ObjectSave(volume)
}
func CreateDeltaBlockBackup(volume *Volume, snapshot *Snapshot, destURL string, sDriver convoydriver.ConvoyDriver) (string, error) {
deltaOps, ok := sDriver.(DeltaBlockBackupOperations)
if !ok {
return "", fmt.Errorf("Driver %s doesn't implemented DeltaBlockBackupOperations interface", sDriver.Name())
}
bsDriver, err := GetObjectStoreDriver(destURL)
if err != nil {
return "", err
}
if err := addVolume(volume, bsDriver); err != nil {
return "", err
}
// Update volume from objectstore
volume, err = loadVolume(volume.UUID, bsDriver)
if err != nil {
return "", err
}
lastBackupUUID := volume.LastBackupUUID
var lastSnapshotUUID string
var lastBackup *Backup
if lastBackupUUID != "" {
lastBackup, err = loadBackup(lastBackupUUID, volume.UUID, bsDriver)
if err != nil {
return "", err
}
lastSnapshotUUID = lastBackup.SnapshotUUID
if lastSnapshotUUID == snapshot.UUID {
//Generate full snapshot if the snapshot has been backed up last time
lastSnapshotUUID = ""
log.Debug("Would create full snapshot metadata")
} else if !deltaOps.HasSnapshot(lastSnapshotUUID, volume.UUID) {
// It's possible that the snapshot in objectstore doesn't exist
// in local storage
lastSnapshotUUID = ""
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_FALLBACK,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_SNAPSHOT: lastSnapshotUUID,
LOG_FIELD_VOLUME: volume.UUID,
}).Debug("Cannot find last snapshot in local storage, would process with full backup")
}
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_START,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_EVENT: LOG_EVENT_COMPARE,
LOG_FIELD_SNAPSHOT: snapshot.UUID,
LOG_FIELD_LAST_SNAPSHOT: lastSnapshotUUID,
}).Debug("Generating snapshot changed blocks metadata")
delta, err := deltaOps.CompareSnapshot(snapshot.UUID, lastSnapshotUUID, volume.UUID)
if err != nil {
return "", err
}
if delta.BlockSize != DEFAULT_BLOCK_SIZE {
return "", fmt.Errorf("Currently doesn't support different block sizes driver other than %v", DEFAULT_BLOCK_SIZE)
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_COMPLETE,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_EVENT: LOG_EVENT_COMPARE,
LOG_FIELD_SNAPSHOT: snapshot.UUID,
LOG_FIELD_LAST_SNAPSHOT: lastSnapshotUUID,
}).Debug("Generated snapshot changed blocks metadata")
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_START,
LOG_FIELD_EVENT: LOG_EVENT_BACKUP,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_SNAPSHOT: snapshot.UUID,
}).Debug("Creating backup")
deltaBackup := &Backup{
UUID: uuid.New(),
VolumeUUID: volume.UUID,
SnapshotUUID: snapshot.UUID,
Blocks: []BlockMapping{},
}
if err := deltaOps.OpenSnapshot(snapshot.UUID, volume.UUID); err != nil {
return "", err
}
defer deltaOps.CloseSnapshot(snapshot.UUID, volume.UUID)
mCounts := len(delta.Mappings)
for m, d := range delta.Mappings {
block := make([]byte, DEFAULT_BLOCK_SIZE)
blkCounts := d.Size / delta.BlockSize
for i := int64(0); i < blkCounts; i++ {
offset := d.Offset + i*delta.BlockSize
log.Debugf("Backup for %v: segment %v/%v, blocks %v/%v", snapshot.UUID, m+1, mCounts, i+1, blkCounts)
err := deltaOps.ReadSnapshot(snapshot.UUID, volume.UUID, offset, block)
if err != nil {
return "", err
}
checksum := util.GetChecksum(block)
blkFile := getBlockFilePath(volume.UUID, checksum)
if bsDriver.FileSize(blkFile) >= 0 {
blockMapping := BlockMapping{
Offset: offset,
BlockChecksum: checksum,
}
deltaBackup.Blocks = append(deltaBackup.Blocks, blockMapping)
log.Debugf("Found existed block match at %v", blkFile)
continue
}
rs, err := util.CompressData(block)
if err != nil {
return "", err
}
if err := bsDriver.Write(blkFile, rs); err != nil {
return "", err
}
log.Debugf("Created new block file at %v", blkFile)
blockMapping := BlockMapping{
Offset: offset,
BlockChecksum: checksum,
}
deltaBackup.Blocks = append(deltaBackup.Blocks, blockMapping)
}
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_COMPLETE,
LOG_FIELD_EVENT: LOG_EVENT_BACKUP,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_SNAPSHOT: snapshot.UUID,
}).Debug("Created snapshot changed blocks")
backup := mergeSnapshotMap(deltaBackup, lastBackup)
backup.SnapshotName = snapshot.Name
backup.SnapshotCreatedAt = snapshot.CreatedTime
backup.CreatedTime = util.Now()
if err := saveBackup(backup, bsDriver); err != nil {
return "", err
}
volume.LastBackupUUID = backup.UUID
if err := saveVolume(volume, bsDriver); err != nil {
return "", err
}
return encodeBackupURL(backup.UUID, volume.UUID, destURL), nil
}
func (d *Driver) CreateVolume(req Request) error {
d.mutex.Lock()
defer d.mutex.Unlock()
var (
size int64
err error
)
id := req.Name
opts := req.Options
backupURL := opts[OPT_BACKUP_URL]
if backupURL != "" {
objVolume, err := objectstore.LoadVolume(backupURL)
if err != nil {
return err
}
if objVolume.Driver != d.Name() {
return fmt.Errorf("Cannot restore backup of %v to %v", objVolume.Driver, d.Name())
}
size, err = d.getSize(opts, objVolume.Size)
if err != nil {
return err
}
if size != objVolume.Size {
return fmt.Errorf("Volume size must match with backup's size")
}
} else {
size, err = d.getSize(opts, d.DefaultVolumeSize)
if err != nil {
return err
}
}
if size%(d.ThinpoolBlockSize*SECTOR_SIZE) != 0 {
return fmt.Errorf("Size must be multiple of block size")
}
volume := d.blankVolume(id)
exists, err := util.ObjectExists(volume)
if err != nil {
return err
}
if exists {
return generateError(logrus.Fields{
LOG_FIELD_VOLUME: id,
}, "Already has volume with specific uuid")
}
devID, err := d.allocateDevID()
if err != nil {
return err
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_START,
LOG_FIELD_EVENT: LOG_EVENT_CREATE,
LOG_FIELD_OBJECT: LOG_OBJECT_VOLUME,
LOG_FIELD_VOLUME: id,
DM_LOG_FIELD_VOLUME_DEVID: devID,
}).Debugf("Creating volume")
err = devicemapper.CreateDevice(d.ThinpoolDevice, devID)
if err != nil {
return err
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_START,
LOG_FIELD_EVENT: LOG_EVENT_ACTIVATE,
LOG_FIELD_OBJECT: LOG_OBJECT_VOLUME,
LOG_FIELD_VOLUME: id,
DM_LOG_FIELD_VOLUME_DEVID: devID,
}).Debugf("Activating device for volume")
err = devicemapper.ActivateDevice(d.ThinpoolDevice, id, devID, uint64(size))
if err != nil {
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_ROLLBACK,
LOG_FIELD_EVENT: LOG_EVENT_REMOVE,
LOG_FIELD_OBJECT: LOG_OBJECT_VOLUME,
LOG_FIELD_VOLUME: id,
DM_LOG_FIELD_VOLUME_DEVID: devID,
}).Debugf("Removing device for volume due to fail to activate")
if err := devicemapper.DeleteDevice(d.ThinpoolDevice, devID); err != nil {
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_FAILURE,
LOG_FIELD_EVENT: LOG_EVENT_REMOVE,
LOG_FIELD_OBJECT: LOG_OBJECT_VOLUME,
LOG_FIELD_VOLUME: id,
DM_LOG_FIELD_VOLUME_DEVID: devID,
}).Debugf("Failed to remove device")
}
return err
}
volume.DevID = devID
volume.Name = id
volume.Size = size
volume.CreatedTime = util.Now()
volume.Snapshots = make(map[string]Snapshot)
if err := util.ObjectSave(volume); err != nil {
return err
}
dev, err := d.GetVolumeDevice(id)
if err != nil {
return err
}
if backupURL == "" {
// format the device
if _, err := util.Execute("mkfs", []string{"-t", "ext4", dev}); err != nil {
return err
}
} else {
if err := objectstore.RestoreDeltaBlockBackup(backupURL, dev); err != nil {
return err
}
}
return nil
}
func (s *daemon) doSnapshotCreate(version string, w http.ResponseWriter, r *http.Request, objs map[string]string) error {
s.GlobalLock.Lock()
defer s.GlobalLock.Unlock()
request := &api.SnapshotCreateRequest{}
if err := decodeRequest(r, request); err != nil {
return err
}
volumeUUID := request.VolumeUUID
if err := util.CheckUUID(volumeUUID); err != nil {
return err
}
snapshotName := request.Name
if snapshotName != "" {
if err := util.CheckName(snapshotName); err != nil {
return err
}
existUUID := s.NameUUIDIndex.Get(snapshotName)
if existUUID != "" {
return fmt.Errorf("Snapshot name %v already associated with %v", snapshotName, existUUID)
}
}
volume := s.loadVolume(volumeUUID)
if volume == nil {
return fmt.Errorf("volume %v doesn't exist", volumeUUID)
}
snapOps, err := s.getSnapshotOpsForVolume(volume)
if err != nil {
return err
}
uuid := uuid.New()
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_PREPARE,
LOG_FIELD_EVENT: LOG_EVENT_CREATE,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_SNAPSHOT: uuid,
LOG_FIELD_VOLUME: volumeUUID,
}).Debug()
if err := snapOps.CreateSnapshot(uuid, volumeUUID); err != nil {
return err
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_COMPLETE,
LOG_FIELD_EVENT: LOG_EVENT_CREATE,
LOG_FIELD_OBJECT: LOG_OBJECT_SNAPSHOT,
LOG_FIELD_SNAPSHOT: uuid,
LOG_FIELD_VOLUME: volumeUUID,
}).Debug()
snapshot := Snapshot{
UUID: uuid,
VolumeUUID: volumeUUID,
Name: snapshotName,
CreatedTime: util.Now(),
}
//TODO: error handling
volume.Snapshots[uuid] = snapshot
if err := s.UUIDIndex.Add(snapshot.UUID); err != nil {
return err
}
if err := s.SnapshotVolumeIndex.Add(snapshot.UUID, volume.UUID); err != nil {
return err
}
if snapshot.Name != "" {
if err := s.NameUUIDIndex.Add(snapshot.Name, snapshot.UUID); err != nil {
return err
}
}
if err := s.saveVolume(volume); err != nil {
return err
}
driverInfo, err := s.getSnapshotDriverInfo(snapshot.UUID, volume)
if err != nil {
return err
}
if request.Verbose {
return writeResponseOutput(w, api.SnapshotResponse{
UUID: snapshot.UUID,
VolumeUUID: snapshot.VolumeUUID,
Name: snapshot.Name,
CreatedTime: snapshot.CreatedTime,
DriverInfo: driverInfo,
})
}
return writeStringResponse(w, snapshot.UUID)
}
func (s *daemon) processVolumeCreate(request *api.VolumeCreateRequest) (*Volume, error) {
volumeName := request.Name
driverName := request.DriverName
existedVolume := s.loadVolumeByName(volumeName)
if existedVolume != nil {
return nil, fmt.Errorf("Volume name %v already associate locally with volume %v ", volumeName, existedVolume.UUID)
}
volumeUUID := uuid.New()
if volumeName == "" {
volumeName = "volume-" + volumeUUID[:8]
for s.NameUUIDIndex.Get(volumeName) != "" {
volumeUUID = uuid.New()
volumeName = "volume-" + volumeUUID[:8]
}
}
if driverName == "" {
driverName = s.DefaultDriver
}
driver, err := s.getDriver(driverName)
if err != nil {
return nil, err
}
volOps, err := driver.VolumeOps()
if err != nil {
return nil, err
}
opts := map[string]string{
convoydriver.OPT_SIZE: strconv.FormatInt(request.Size, 10),
convoydriver.OPT_BACKUP_URL: util.UnescapeURL(request.BackupURL),
convoydriver.OPT_VOLUME_NAME: request.Name,
convoydriver.OPT_VOLUME_ID: request.DriverVolumeID,
convoydriver.OPT_VOLUME_TYPE: request.Type,
convoydriver.OPT_VOLUME_IOPS: strconv.FormatInt(request.IOPS, 10),
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_PREPARE,
LOG_FIELD_EVENT: LOG_EVENT_CREATE,
LOG_FIELD_OBJECT: LOG_OBJECT_VOLUME,
LOG_FIELD_VOLUME: volumeUUID,
LOG_FIELD_VOLUME_NAME: volumeName,
LOG_FIELD_OPTS: opts,
}).Debug()
if err := volOps.CreateVolume(volumeUUID, opts); err != nil {
return nil, err
}
log.WithFields(logrus.Fields{
LOG_FIELD_REASON: LOG_REASON_COMPLETE,
LOG_FIELD_EVENT: LOG_EVENT_CREATE,
LOG_FIELD_OBJECT: LOG_OBJECT_VOLUME,
LOG_FIELD_VOLUME: volumeUUID,
}).Debug("Created volume")
volume := &Volume{
UUID: volumeUUID,
Name: volumeName,
DriverName: driverName,
FileSystem: "ext4",
CreatedTime: util.Now(),
Snapshots: make(map[string]Snapshot),
}
if err := s.saveVolume(volume); err != nil {
return nil, err
}
if err := s.UUIDIndex.Add(volume.UUID); err != nil {
return nil, err
}
if volume.Name != "" {
if err := s.NameUUIDIndex.Add(volume.Name, volume.UUID); err != nil {
return nil, err
}
}
return volume, nil
}
func doBackupCreate(c *cli.Context) error {
var (
err error
backingFile *replica.BackingFile
)
destURL, err := util.GetFlag(c, "dest", true, err)
if err != nil {
return err
}
snapshotName, err := getName(c, "", true)
if err != nil {
return err
}
volumeName, err := util.GetFlag(c, "volume", true, err)
if err != nil {
return err
}
dir, err := os.Getwd()
if err != nil {
return err
}
volumeInfo, err := replica.ReadInfo(dir)
if err != nil {
return err
}
if volumeInfo.BackingFileName != "" {
backingFileName := volumeInfo.BackingFileName
if _, err := os.Stat(backingFileName); err != nil {
return err
}
backingFile, err = openBackingFile(backingFileName)
if err != nil {
return err
}
}
replicaBackup := replica.NewBackup(backingFile)
volume := &objectstore.Volume{
Name: volumeName,
Driver: DRIVERNAME,
Size: volumeInfo.Size,
CreatedTime: util.Now(),
}
snapshot := &objectstore.Snapshot{
Name: snapshotName,
CreatedTime: util.Now(),
}
log.Debugf("Starting backup for %v, snapshot %v, dest %v", volume, snapshot, destURL)
backupURL, err := objectstore.CreateDeltaBlockBackup(volume, snapshot, destURL, replicaBackup)
if err != nil {
return err
}
fmt.Println(backupURL)
return nil
}