Exemple #1
0
// createAndRetrieveFiles attempts to create a migration's state file,
// if it does not exist, and retrieve the latest version from the api
func (runner *runner) createAndRetrieveFiles(currentMigration *migration.Migration) error {
	if _, err := os.Stat(currentMigration.FilesDir); err != nil {
		if os.IsNotExist(err) {
			err := os.Mkdir(currentMigration.FilesDir, 0777)
			if err != nil {
				glog.Errorf("mig_id=%d: error creating pt-osc log directory '%s' (error: %s)", currentMigration.Id, currentMigration.FilesDir, err)
				return ErrFiles
			}
		} else {
			glog.Errorf("mig_id=%d: error stat-ing pt-osc log directory '%s' (error: %s)", currentMigration.Id, currentMigration.FilesDir, err)
			return ErrFiles
		}
	}

	// try to retrieve a statefile from the api
	urlParams := map[string]string{
		"migration_id": strconv.Itoa(currentMigration.Id),
		"file_type":    STATE_FILE_TYPE,
	}
	stateFile, restErr := runner.RestClient.GetFile(urlParams)
	if restErr == nil && stateFile["contents"] != nil && len(stateFile["contents"].(string)) > 0 {
		err := currentMigration.WriteStateFile([]byte(stateFile["contents"].(string)))
		if err != nil {
			glog.Errorf("mig_id=%d: error writing to statefile '%s' (error: %s)", currentMigration.Id, currentMigration.StateFile, err)
			return ErrFiles
		}
	}
	return nil
}
Exemple #2
0
func (runner *runner) postStateFile(migration *migration.Migration) {
	data, err := migration.ReadStateFile()
	if err != nil {
		glog.Warningf("Could not read state file (%s)", err)
	} else {
		urlParams := make(map[string]string)
		urlParams["migration_id"] = strconv.Itoa(migration.Id)
		urlParams["file_type"] = STATE_FILE_TYPE
		urlParams["contents"] = string(data[:])

		_, err = runner.RestClient.WriteFile(urlParams)
		if err != nil {
			glog.Errorf("Error POSTing to write_file endpoint (error: %s)", err)
		} else {
			glog.Infof("Sent state file to write_file endpoint")
		}
	}
}
Exemple #3
0
// execPtOsc shells out and uses pt-osc to actually run a migration.
func (runner *runner) execPtOsc(currentMigration *migration.Migration,
	ptOscOptionGenerator commandOptionGenerator, copyPercentChan chan int, unstageDone bool) (bool, error) {

	// unstageDone = whether or not unstagedMigrationsWaitGroup has already had .Done() called on it
	defer func() {
		if !unstageDone {
			unstagedMigrationsWaitGroup.Done()
		}
	}()
	canceled := false

	var fileRoutineWaitGroup sync.WaitGroup

	err := runner.createAndRetrieveFiles(currentMigration)
	if err != nil {
		return canceled, err
	}

	ptOscLogFile, ptOscLogWriter, err := setupLogWriter(currentMigration.LogFile)
	if err != nil {
		glog.Errorf("mig_id=%d: error creating pt-osc log file '%s' (error: %s)", currentMigration.Id, currentMigration.LogFile, err)
		return canceled, ErrPtOscExec
	}
	defer ptOscLogFile.Close()

	// generate the pt-osc command to run
	commandOptions := ptOscOptionGenerator(currentMigration)
	glog.Infof("mig_id=%d: Running %s %v", currentMigration.Id, runner.PtOscPath, strings.Join(commandOptions, " "))
	cmd := exec.Command(runner.PtOscPath, commandOptions...)

	// capture stdout and stderr of the command
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		glog.Errorf("mig_id=%d: error getting stdout pipe for pt-osc exec (error: %s)", currentMigration.Id, err)
		return canceled, ErrPtOscExec
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		glog.Errorf("mig_id=%d: error getting stderr pipe for pt-osc exec (error: %s)", currentMigration.Id, err)
		return canceled, ErrPtOscExec
	}

	// start the pt-osc comand
	if err := cmd.Start(); err != nil {
		glog.Errorf("mig_id=%d: error starting pt-osc exec (error: %s)", currentMigration.Id, err)
		return canceled, ErrPtOscExec
	}

	// setup a channel and goroutine for logging output of stdout/stderr
	ptOscLogChan := make(chan string)
	fileRoutineWaitGroup.Add(1)
	fileSyncWaitGroup.Add(1)
	go runner.writeToPtOscLog(ptOscLogWriter, ptOscLogChan, writeLineToPtOscLog, currentMigration.Id, &fileRoutineWaitGroup)

	// setup a goroutine for sending statefiles to the api
	ptOscStateFileChan := make(chan string)
	fileRoutineWaitGroup.Add(1)
	fileSyncWaitGroup.Add(1)
	go runner.syncStateFile(currentMigration, ptOscStateFileChan, &fileRoutineWaitGroup)

	// setup goroutines for watching stdout/stderr of the command
	stdoutErrChan := make(chan error)
	stderrErrChan := make(chan error)
	go currentMigration.WatchMigrationStdout(stdout, stdoutErrChan, ptOscLogChan)
	if currentMigration.Status == migration.RunMigrationStatus {
		// setup a goroutine to continually update the % copied of the migration
		fileRoutineWaitGroup.Add(1)
		fileSyncWaitGroup.Add(1)
		go runner.updateMigrationCopyPercentage(currentMigration, copyPercentChan, &fileRoutineWaitGroup)
		go currentMigration.WatchMigrationCopyStderr(stderr, copyPercentChan, stderrErrChan, ptOscLogChan)
	} else {
		go currentMigration.WatchMigrationStderr(stderr, stderrErrChan, ptOscLogChan)
	}

	// save the pid of the pt-osc process
	currentMigration.Pid = cmd.Process.Pid
	glog.Infof("mig_id=%d: pt-osc pid for status %d is %d.", currentMigration.Id, currentMigration.Status, currentMigration.Pid)

	// add the migration id and pid to the running migration map
	runningMigMutex.Lock()
	runningMigrations[currentMigration.Id] = currentMigration.Pid
	runningMigMutex.Unlock()
	if !unstageDone {
		unstageDone = true
		unstagedMigrationsWaitGroup.Done()
	}

	// wait for both stdout and stderr error channels to receive a signal
	stdoutErr := <-stdoutErrChan
	stderrErr := <-stderrErrChan
	close(ptOscLogChan)
	close(ptOscStateFileChan)

	// remove the migration id from the running migration map
	runningMigMutex.Lock()
	delete(runningMigrations, currentMigration.Id)
	runningMigMutex.Unlock()

	// get the exit status of the command. if it was sent a SIGKILL (most likely
	// by another goroutine) we want to know because we will treat it differently
	failed := false
	if err := cmd.Wait(); err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				exitSignal := status.Signal().String()
				glog.Infof("mig_id=%d: exit signal was %s.", currentMigration.Id, exitSignal)
				if (exitSignal == SIGKILLSignal) && (currentMigration.Status == migration.RunMigrationStatus) {
					// was killed
					glog.Infof("mig_id=%d: migration must have been canceled", currentMigration.Id)
					canceled = true
				} else if currentMigration.Status == migration.RunMigrationStatus {
					// died for an unexpected reason
					glog.Infof("mig_id=%d: migration died for an unexpected reason", currentMigration.Id)
					failed = true
				}
			}
		}
	} else {
		if (stderrErr == nil) && (currentMigration.Status == migration.RunMigrationStatus) {
			// wasn't killed. copy completed 100%
			glog.Infof("mig_id=%d: updating migration with copy percentage of 100", currentMigration.Id)
			copyPercentChan <- 100
		}
	}

	if copyPercentChan != nil {
		glog.Infof("Closing copy percent channel")
		close(copyPercentChan)
	}

	fileRoutineWaitGroup.Wait() // wait for go routines to finish

	// favor returning error from unexpected failure, then error from stderr,
	// and lastly error from stdout
	if failed {
		return canceled, ErrUnexpectedExit
	} else if stderrErr != nil {
		return canceled, stderrErr
	} else if stdoutErr != nil {
		return canceled, stdoutErr
	}

	return canceled, nil
}