func (w *worker) startInstance(retryIfFails bool) error {
w.wc.logger.Info("worker.startInstance(): w=", w)
if err := w.setSAI(WS_STARTING, spi.NO_APP_ID, spi.NO_INST_ID); err != nil {
return err
}
ic := w.instanceConfig()
var inst spi.Instance
for inst == nil && w.checkState() == nil {
var err error
inst, err = w.wc.ip.NewInstance(ic)
if err != nil {
if !retryIfFails {
return err
}
if err := w.checkState(); err != nil {
return err
}
runtime.Gosched()
continue
}
}
if err := w.setSAI(WS_STARTED, spi.NO_APP_ID, inst.Id()); err != nil {
return err
}
w.wc.logger.Info("worker.startInstance(): Started. w=", w)
return nil
}
func (w *worker) Recover() error {
logger := w.wc.logger
logger.Info("worker.Recover(): w=", w)
if err := w.setS(WS_RECOVERING); err != nil {
return err
}
var instance spi.Instance
for instance == nil {
instance := w.getInstanceForRecover()
if instance == nil {
return errors.New(fmt.Sprint("Cannot recover worker id=", w.Id(), " looks like state is lost"))
}
oldInstance := w.instance()
if err := w.setSAI(WS_RECOVERING, w.desc.Aid, instance.Id()); err != nil {
return err
}
if oldInstance != nil {
logger.Info("worker.Recover(): stopping old instance")
go func() {
oldInstance.Stop()
}()
}
app := w.wc.ap.Application(w.desc.Aid)
if app == nil {
logger.Warn("worker.Recover(): cannot find application, will move worker to STARTED state")
go func() {
w.startInstance(true)
}()
return errors.New("Cannot recover, application is not found for worker=" + w.String())
}
if err := app.Deploy(instance); err != nil {
go func(instance spi.Instance) {
instance.Stop()
}(instance)
w.wc.logger.Warn("worker.Recover(): failed to deploy application to the instance: ", err)
instance = nil
}
}
if err := w.setSAI(WS_DEPLOYED, w.desc.Aid, instance.Id()); err != nil {
return err
}
return nil
}