func (r *replication) ensureInReality(
node types.NodeName,
timeoutCh <-chan struct{},
nodeLogger logging.Logger,
targetSHA string,
) error {
for {
select {
case <-r.quitCh:
return errQuit
case <-timeoutCh:
return errTimeout
case <-r.replicationCancelledCh:
return errCancelled
case <-time.After(5 * time.Second):
man, err := r.queryReality(node)
if err == pods.NoCurrentManifest {
// if the pod key doesn't exist yet, that's okay just wait longer
} else if err != nil {
nodeLogger.WithErrorAndFields(err, logrus.Fields{
"node": node,
}).Errorln("Could not read reality for pod manifest")
} else {
receivedSHA, _ := man.SHA()
if receivedSHA == targetSHA {
nodeLogger.NoFields().Infoln("Node is current")
return nil
} else {
nodeLogger.WithFields(logrus.Fields{"current": receivedSHA, "target": targetSHA}).Infoln("Waiting for current")
}
}
}
}
}
func (dsf *Farm) handleSessionExpiry(dsFields ds_fields.DaemonSet, dsLogger logging.Logger, err error) {
dsLogger.WithFields(logrus.Fields{
"ID": dsFields.ID,
"Name": dsFields.Name,
"NodeSelector": dsFields.NodeSelector.String(),
"PodID": dsFields.PodID,
}).WithError(err).Errorln("Got error while locking daemon set in ds farm - session may be expired")
}
func (p *Preparer) tryRunHooks(hookType hooks.HookType, pod hooks.Pod, manifest *pods.Manifest, logger logging.Logger) {
err := p.hooks.RunHookType(hookType, pod, manifest)
if err != nil {
logger.WithFields(logrus.Fields{
"err": err,
"hooks": hookType,
}).Warnln("Could not run hooks")
}
}
func addHooks(preparerConfig *PreparerConfig, logger logging.Logger) {
for _, dest := range preparerConfig.ExtraLogDestinations {
logger.WithFields(logrus.Fields{
"type": dest.Type,
"path": dest.Path,
}).Infoln("Adding log destination")
logger.AddHook(dest.Type, dest.Path)
}
}
func addHooks(preparerConfig *PreparerConfig, logger logging.Logger) {
for _, dest := range preparerConfig.ExtraLogDestinations {
logger.WithFields(logrus.Fields{
"type": dest.Type,
"path": dest.Path,
}).Infoln("Adding log destination")
if err := logger.AddHook(dest.Type, dest.Path); err != nil {
logger.WithError(err).Errorf("Unable to add log hook. Proceeding.")
}
}
}
// check if a manifest satisfies the authorization requirement of this preparer
func (p *Preparer) authorize(manifest manifest.Manifest, logger logging.Logger) bool {
err := p.authPolicy.AuthorizeApp(manifest, logger)
if err != nil {
if err, ok := err.(auth.Error); ok {
logger.WithFields(err.Fields).Errorln(err)
} else {
logger.NoFields().Errorln(err)
}
return false
}
return true
}
func (r *replication) ensureHealthy(
node types.NodeName,
timeoutCh <-chan struct{},
nodeLogger logging.Logger,
aggregateHealth *podHealth,
) error {
for {
select {
case <-r.quitCh:
r.logger.Infoln("Caught quit signal during ensureHealthy")
return errQuit
case <-timeoutCh:
r.logger.Infoln("Caught node timeout signal during ensureHealthy")
return errTimeout
case <-r.replicationCancelledCh:
r.logger.Infoln("Caught cancellation signal during ensureHealthy")
return errCancelled
case <-time.After(time.Duration(*ensureHealthyPeriodMillis) * time.Millisecond):
res, ok := aggregateHealth.GetHealth(node)
if !ok {
nodeLogger.WithFields(logrus.Fields{
"node": node,
}).Errorln("Could not get health, retrying")
// Zero res should be treated like "critical"
}
id := res.ID
status := res.Status
// treat an empty threshold as "passing"
threshold := health.Passing
if r.threshold != "" {
threshold = r.threshold
}
// is this status less than the threshold?
if health.Compare(status, threshold) < 0 {
nodeLogger.WithFields(logrus.Fields{"check": id, "health": status}).Infoln("Node is not healthy")
} else {
r.logger.WithField("node", node).Infoln("Node is current and healthy")
return nil
}
}
}
}
func runDirectory(dirpath string, environment []string, logger logging.Logger) error {
entries, err := ioutil.ReadDir(dirpath)
if os.IsNotExist(err) {
logger.WithField("dir", dirpath).Debugln("Hooks not set up")
return nil
}
if err != nil {
return err
}
for _, f := range entries {
fullpath := path.Join(dirpath, f.Name())
executable := (f.Mode() & 0111) != 0
if !executable {
logger.WithField("path", fullpath).Warnln("Hook is not executable")
continue
}
if f.IsDir() {
continue
}
cmd := exec.Command(fullpath)
hookOut := &bytes.Buffer{}
cmd.Stdout = hookOut
cmd.Stderr = hookOut
cmd.Env = environment
err := cmd.Run()
if err != nil {
logger.WithErrorAndFields(err, logrus.Fields{
"path": fullpath,
"output": hookOut.String(),
}).Warnf("Could not execute hook %s", f.Name())
} else {
logger.WithFields(logrus.Fields{
"path": fullpath,
"output": hookOut.String(),
}).Debugln("Executed hook")
}
}
return nil
}
func (p *Preparer) installAndLaunchPod(newManifest *pods.Manifest, pod Pod, logger logging.Logger) bool {
// do not remove the logger argument, it's not the same as p.Logger
// get currently running pod to compare with the new pod
realityPath := kp.RealityPath(p.node, newManifest.ID())
currentManifest, _, err := p.store.Pod(realityPath)
currentSHA := ""
if currentManifest != nil {
currentSHA, _ = currentManifest.SHA()
}
newSHA, _ := newManifest.SHA()
// if new or the manifest is different, launch
newOrDifferent := (err == pods.NoCurrentManifest) || (currentSHA != newSHA)
if newOrDifferent {
logger.WithFields(logrus.Fields{
"old_sha": currentSHA,
"sha": newSHA,
"pod": newManifest.ID(),
}).Infoln("SHA is new or different from old, will update")
}
// if the old manifest is corrupted somehow, re-launch since we don't know if this is an update.
problemReadingCurrentManifest := (err != nil && err != pods.NoCurrentManifest)
if problemReadingCurrentManifest {
logger.WithFields(logrus.Fields{
"sha": newSHA,
"inner_err": err,
}).Errorln("Current manifest not readable, will relaunch")
}
if newOrDifferent || problemReadingCurrentManifest {
p.tryRunHooks(hooks.BEFORE_INSTALL, pod, newManifest, logger)
err = pod.Install(newManifest)
if err != nil {
// install failed, abort and retry
logger.WithFields(logrus.Fields{
"err": err,
}).Errorln("Install failed")
return false
}
err = pod.Verify(newManifest, p.authPolicy)
if err != nil {
logger.WithField("err", err).Errorln("Pod digest verification failed")
p.tryRunHooks(hooks.AFTER_AUTH_FAIL, pod, newManifest, logger)
return false
}
p.tryRunHooks(hooks.AFTER_INSTALL, pod, newManifest, logger)
err = p.store.RegisterService(*newManifest, p.caPath)
if err != nil {
logger.WithField("err", err).Errorln("Service registration failed")
return false
}
if currentManifest != nil {
success, err := pod.Halt(currentManifest)
if err != nil {
logger.WithField("err", err).Errorln("Pod halt failed")
} else if !success {
logger.NoFields().Warnln("One or more launchables did not halt successfully")
}
}
ok, err := pod.Launch(newManifest)
if err != nil {
logger.WithFields(logrus.Fields{
"err": err,
}).Errorln("Launch failed")
} else {
duration, err := p.store.SetPod(realityPath, *newManifest)
if err != nil {
logger.WithFields(logrus.Fields{
"err": err,
"duration": duration,
}).Errorln("Could not set pod in reality store")
}
p.tryRunHooks(hooks.AFTER_LAUNCH, pod, newManifest, logger)
}
return err == nil && ok
}
// TODO: shut down removed launchables between pod versions.
return true
}
// Populates the given directory with executor scripts for each launch script of
// the given pod, which must be installed. Any orphaned executor scripts (from a
// past install, but no longer present in this pod) will be cleaned out.
func InstallHookScripts(dir string, hookPod *pods.Pod, manifest pods.Manifest, logger logging.Logger) error {
err := os.MkdirAll(dir, 0755)
if err != nil {
return err
}
// TODO: globbing based on the structure of the name is gross, each hook pod
// should have its own dir of scripts and running hooks should iterate over
// the directories
rmPattern := filepath.Join(dir, fmt.Sprintf("%s__*", hookPod.Id))
matches, err := filepath.Glob(rmPattern)
if err != nil {
// error return from filepath.Glob is guaranteed to be ErrBadPattern
return util.Errorf("error while removing old hook scripts: pattern %q is malformed", rmPattern)
}
for _, match := range matches {
err = os.Remove(match)
if err != nil {
logger.WithErrorAndFields(err, logrus.Fields{"script_path": match}).Errorln("Could not remove old hook script")
}
}
launchables, err := hookPod.Launchables(manifest)
if err != nil {
return err
}
for _, launchable := range launchables {
if launchable.Type() != "hoist" {
logger.WithFields(logrus.Fields{
"id": launchable.ID(),
"type": launchable.Type(),
}).Errorln("hook disabled: unsupported launchable type")
continue
}
executables, err := launchable.Executables(runit.DefaultBuilder)
if err != nil {
return err
}
for _, executable := range executables {
// Write a script to the event directory that executes the pod's executables
// with the correct environment for that pod.
scriptPath := filepath.Join(dir, executable.Service.Name)
file, err := os.OpenFile(scriptPath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0744)
defer file.Close()
if err != nil {
logger.WithErrorAndFields(err, logrus.Fields{"script_path": scriptPath}).Errorln("Could not open new hook script")
}
err = executable.WriteExecutor(file)
if err != nil {
logger.WithErrorAndFields(err, logrus.Fields{"script_path": scriptPath}).Errorln("Could not write new hook script")
}
}
// for convenience as we do with regular launchables, make these ones
// current under the launchable directory
err = launchable.MakeCurrent()
if err != nil {
logger.WithErrorAndFields(err, logrus.Fields{"launchable_id": launchable.ID()}).
Errorln("Could not set hook launchable to current")
}
}
return nil
}
