Example #1
0
func CreateSingleFileBackup(volume *Volume, snapshot *Snapshot, filePath, destURL string) (string, error) {
	driver, err := GetObjectStoreDriver(destURL)
	if err != nil {
		return "", err
	}

	if err := addVolume(volume, driver); err != nil {
		return "", err
	}

	volume, err = loadVolume(volume.Name, driver)
	if err != nil {
		return "", err
	}

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:   LOG_REASON_START,
		LOG_FIELD_EVENT:    LOG_EVENT_BACKUP,
		LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_SNAPSHOT: snapshot.Name,
		LOG_FIELD_FILEPATH: filePath,
	}).Debug("Creating backup")

	backup := &Backup{
		Name:              util.GenerateName("backup"),
		VolumeName:        volume.Name,
		SnapshotName:      snapshot.Name,
		SnapshotCreatedAt: snapshot.CreatedTime,
	}
	backup.SingleFile.FilePath = getSingleFileBackupFilePath(backup)

	if err := driver.Upload(filePath, backup.SingleFile.FilePath); err != nil {
		return "", err
	}

	backup.CreatedTime = util.Now()
	if err := saveBackup(backup, driver); err != nil {
		return "", err
	}

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:   LOG_REASON_COMPLETE,
		LOG_FIELD_EVENT:    LOG_EVENT_BACKUP,
		LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_SNAPSHOT: snapshot.Name,
	}).Debug("Created backup")

	return encodeBackupURL(backup.Name, volume.Name, destURL), nil
}
Example #2
0
func (s *daemon) processVolumeCreate(request *api.VolumeCreateRequest) (*Volume, error) {
	volumeName := request.Name
	driverName := request.DriverName

	if volumeName == "" {
		volumeName = util.GenerateName("volume")
		for s.NameUUIDIndex.Get(volumeName) != "" {
			volumeName = util.GenerateName("volume")
		}
	} else {
		if s.NameUUIDIndex.Get(volumeName) != "" {
			return nil, fmt.Errorf("Volume %v already exists ", volumeName)
		}
	}

	if driverName == "" {
		driverName = s.DefaultDriver
	}
	driver, err := s.getDriver(driverName)
	if err != nil {
		return nil, err
	}
	volOps, err := driver.VolumeOps()
	if err != nil {
		return nil, err
	}

	req := Request{
		Name: volumeName,
		Options: map[string]string{
			OPT_SIZE:             strconv.FormatInt(request.Size, 10),
			OPT_BACKUP_URL:       util.UnescapeURL(request.BackupURL),
			OPT_VOLUME_NAME:      volumeName,
			OPT_VOLUME_DRIVER_ID: request.DriverVolumeID,
			OPT_VOLUME_TYPE:      request.Type,
			OPT_VOLUME_IOPS:      strconv.FormatInt(request.IOPS, 10),
			OPT_PREPARE_FOR_VM:   strconv.FormatBool(request.PrepareForVM),
		},
	}
	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON: LOG_REASON_PREPARE,
		LOG_FIELD_EVENT:  LOG_EVENT_CREATE,
		LOG_FIELD_OBJECT: LOG_OBJECT_VOLUME,
		LOG_FIELD_VOLUME: volumeName,
		LOG_FIELD_OPTS:   req.Options,
	}).Debug()
	if err := volOps.CreateVolume(req); err != nil {
		return nil, err
	}
	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON: LOG_REASON_COMPLETE,
		LOG_FIELD_EVENT:  LOG_EVENT_CREATE,
		LOG_FIELD_OBJECT: LOG_OBJECT_VOLUME,
		LOG_FIELD_VOLUME: volumeName,
	}).Debug("Created volume")

	volume := &Volume{
		Name:       volumeName,
		DriverName: driverName,
	}

	if err := s.NameUUIDIndex.Add(volumeName, "exists"); err != nil {
		return nil, err
	}
	return volume, nil
}
Example #3
0
func (s *daemon) doSnapshotCreate(version string, w http.ResponseWriter, r *http.Request, objs map[string]string) error {
	request := &api.SnapshotCreateRequest{}
	if err := decodeRequest(r, request); err != nil {
		return err
	}
	volumeName := request.VolumeName
	if err := util.CheckName(volumeName); err != nil {
		return err
	}
	volume := s.getVolume(volumeName)
	if volume == nil {
		return fmt.Errorf("volume %v doesn't exist", volumeName)
	}

	snapshotName := request.Name
	if snapshotName != "" {
		if err := util.CheckName(snapshotName); err != nil {
			return err
		}
		existName := s.NameUUIDIndex.Get(snapshotName)
		if existName != "" {
			return fmt.Errorf("Snapshot name %v already exists", snapshotName)
		}
	} else {
		snapshotName = util.GenerateName("snapshot")
		for s.NameUUIDIndex.Get(snapshotName) != "" {
			snapshotName = util.GenerateName("snapshot")
		}
	}

	snapOps, err := s.getSnapshotOpsForVolume(volume)
	if err != nil {
		return err
	}

	req := Request{
		Name: snapshotName,
		Options: map[string]string{
			OPT_VOLUME_NAME: volumeName,
		},
	}

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:   LOG_REASON_PREPARE,
		LOG_FIELD_EVENT:    LOG_EVENT_CREATE,
		LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_SNAPSHOT: snapshotName,
		LOG_FIELD_VOLUME:   volumeName,
	}).Debug()
	if err := snapOps.CreateSnapshot(req); err != nil {
		return err
	}
	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:   LOG_REASON_COMPLETE,
		LOG_FIELD_EVENT:    LOG_EVENT_CREATE,
		LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_SNAPSHOT: snapshotName,
		LOG_FIELD_VOLUME:   volumeName,
	}).Debug()

	//TODO: error handling
	if err := s.SnapshotVolumeIndex.Add(snapshotName, volume.Name); err != nil {
		return err
	}
	if err := s.NameUUIDIndex.Add(snapshotName, "exists"); err != nil {
		return err
	}
	driverInfo, err := s.getSnapshotDriverInfo(snapshotName, volume)
	if err != nil {
		return err
	}
	if request.Verbose {
		return writeResponseOutput(w, api.SnapshotResponse{
			Name:        snapshotName,
			VolumeName:  volume.Name,
			CreatedTime: driverInfo[OPT_SNAPSHOT_CREATED_TIME],
			DriverInfo:  driverInfo,
		})
	}
	return writeStringResponse(w, snapshotName)
}
Example #4
0
func CreateDeltaBlockBackup(volume *Volume, snapshot *Snapshot, destURL string, sDriver convoydriver.ConvoyDriver) (string, error) {
	deltaOps, ok := sDriver.(DeltaBlockBackupOperations)
	if !ok {
		return "", fmt.Errorf("Driver %s doesn't implemented DeltaBlockBackupOperations interface", sDriver.Name())
	}

	bsDriver, err := GetObjectStoreDriver(destURL)
	if err != nil {
		return "", err
	}

	if err := addVolume(volume, bsDriver); err != nil {
		return "", err
	}

	// Update volume from objectstore
	volume, err = loadVolume(volume.Name, bsDriver)
	if err != nil {
		return "", err
	}

	lastBackupName := volume.LastBackupName

	var lastSnapshotName string
	var lastBackup *Backup
	if lastBackupName != "" {
		lastBackup, err = loadBackup(lastBackupName, volume.Name, bsDriver)
		if err != nil {
			return "", err
		}

		lastSnapshotName = lastBackup.SnapshotName
		if lastSnapshotName == snapshot.Name {
			//Generate full snapshot if the snapshot has been backed up last time
			lastSnapshotName = ""
			log.Debug("Would create full snapshot metadata")
		} else if !deltaOps.HasSnapshot(lastSnapshotName, volume.Name) {
			// It's possible that the snapshot in objectstore doesn't exist
			// in local storage
			lastSnapshotName = ""
			log.WithFields(logrus.Fields{
				LOG_FIELD_REASON:   LOG_REASON_FALLBACK,
				LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
				LOG_FIELD_SNAPSHOT: lastSnapshotName,
				LOG_FIELD_VOLUME:   volume.Name,
			}).Debug("Cannot find last snapshot in local storage, would process with full backup")
		}
	}

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:        LOG_REASON_START,
		LOG_FIELD_OBJECT:        LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_EVENT:         LOG_EVENT_COMPARE,
		LOG_FIELD_SNAPSHOT:      snapshot.Name,
		LOG_FIELD_LAST_SNAPSHOT: lastSnapshotName,
	}).Debug("Generating snapshot changed blocks metadata")
	delta, err := deltaOps.CompareSnapshot(snapshot.Name, lastSnapshotName, volume.Name)
	if err != nil {
		return "", err
	}
	if delta.BlockSize != DEFAULT_BLOCK_SIZE {
		return "", fmt.Errorf("Currently doesn't support different block sizes driver other than %v", DEFAULT_BLOCK_SIZE)
	}
	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:        LOG_REASON_COMPLETE,
		LOG_FIELD_OBJECT:        LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_EVENT:         LOG_EVENT_COMPARE,
		LOG_FIELD_SNAPSHOT:      snapshot.Name,
		LOG_FIELD_LAST_SNAPSHOT: lastSnapshotName,
	}).Debug("Generated snapshot changed blocks metadata")

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:   LOG_REASON_START,
		LOG_FIELD_EVENT:    LOG_EVENT_BACKUP,
		LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_SNAPSHOT: snapshot.Name,
	}).Debug("Creating backup")

	deltaBackup := &Backup{
		Name:         util.GenerateName("backup"),
		VolumeName:   volume.Name,
		SnapshotName: snapshot.Name,
		Blocks:       []BlockMapping{},
	}
	if err := deltaOps.OpenSnapshot(snapshot.Name, volume.Name); err != nil {
		return "", err
	}
	defer deltaOps.CloseSnapshot(snapshot.Name, volume.Name)
	mCounts := len(delta.Mappings)
	for m, d := range delta.Mappings {
		block := make([]byte, DEFAULT_BLOCK_SIZE)
		blkCounts := d.Size / delta.BlockSize
		for i := int64(0); i < blkCounts; i++ {
			offset := d.Offset + i*delta.BlockSize
			log.Debugf("Backup for %v: segment %v/%v, blocks %v/%v", snapshot.Name, m+1, mCounts, i+1, blkCounts)
			err := deltaOps.ReadSnapshot(snapshot.Name, volume.Name, offset, block)
			if err != nil {
				return "", err
			}
			checksum := util.GetChecksum(block)
			blkFile := getBlockFilePath(volume.Name, checksum)
			if bsDriver.FileSize(blkFile) >= 0 {
				blockMapping := BlockMapping{
					Offset:        offset,
					BlockChecksum: checksum,
				}
				deltaBackup.Blocks = append(deltaBackup.Blocks, blockMapping)
				log.Debugf("Found existed block match at %v", blkFile)
				continue
			}

			rs, err := util.CompressData(block)
			if err != nil {
				return "", err
			}

			if err := bsDriver.Write(blkFile, rs); err != nil {
				return "", err
			}
			log.Debugf("Created new block file at %v", blkFile)

			blockMapping := BlockMapping{
				Offset:        offset,
				BlockChecksum: checksum,
			}
			deltaBackup.Blocks = append(deltaBackup.Blocks, blockMapping)
		}
	}

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:   LOG_REASON_COMPLETE,
		LOG_FIELD_EVENT:    LOG_EVENT_BACKUP,
		LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_SNAPSHOT: snapshot.Name,
	}).Debug("Created snapshot changed blocks")

	backup := mergeSnapshotMap(deltaBackup, lastBackup)
	backup.SnapshotName = snapshot.Name
	backup.SnapshotCreatedAt = snapshot.CreatedTime
	backup.CreatedTime = util.Now()

	if err := saveBackup(backup, bsDriver); err != nil {
		return "", err
	}

	volume.LastBackupName = backup.Name
	if err := saveVolume(volume, bsDriver); err != nil {
		return "", err
	}

	return encodeBackupURL(backup.Name, volume.Name, destURL), nil
}