func installConsul(consulPod *pods.Pod, consulManifest manifest.Manifest, registryURL *url.URL) error {
// Inject servicebuilder?
err := consulPod.Install(consulManifest, auth.NopVerifier(), artifact.NewRegistry(registryURL, uri.DefaultFetcher, osversion.DefaultDetector))
if err != nil {
return util.Errorf("Can't install Consul, aborting: %s", err)
}
ok, err := consulPod.Launch(consulManifest)
if err != nil || !ok {
return util.Errorf("Can't launch Consul, aborting: %s", err)
}
return nil
}
func InstallConsul(consulPod *pods.Pod, consulManifest *pods.Manifest) error {
// Inject servicebuilder?
err := consulPod.Install(consulManifest)
if err != nil {
return util.Errorf("Can't install Consul, aborting: %s", err)
}
ok, err := consulPod.Launch(consulManifest)
if err != nil || !ok {
return util.Errorf("Can't launch Consul, aborting: %s", err)
}
return nil
}
func (l *HookListener) writeHook(event string, hookPod *pods.Pod, manifest *pods.Manifest) error {
eventExecDir := path.Join(l.ExecDir, event)
err := os.MkdirAll(eventExecDir, 0755)
if err != nil {
return util.Errorf("Couldn't make event exec dir %s", eventExecDir)
}
launchables, err := hookPod.Launchables(manifest)
if err != nil {
return err
}
// First remove any pre-existing hooks for that pod. Note that this is gross
// and that we should have hooks recurse into subfolders.
podHookPattern := path.Join(eventExecDir, fmt.Sprintf("%s__*", hookPod.Id))
matches, err := filepath.Glob(podHookPattern)
if err != nil {
return util.Errorf("Couldn't find files using pattern %s in %s: %s", podHookPattern, eventExecDir, err)
}
for _, match := range matches {
err = os.Remove(match)
if err != nil {
l.Logger.WithField("err", err).Warnln("Could not remove old hook")
}
}
// For every launchable in the manifest, link its executable to the hook directory.
for _, launchable := range launchables {
// warn if a hook has a cgroup configured - it will be ignored
emptyCgroup := cgroups.Config{Name: launchable.CgroupConfig.Name}
if launchable.CgroupConfig != emptyCgroup {
l.Logger.WithField("hook_launchable_id", launchable.Id).Warnln("Hook cgroup will be ignored")
}
executables, err := launchable.Executables(runit.DefaultBuilder)
if err != nil {
return err
}
for _, executable := range executables {
// Write a script to the event directory that executes the pod's executables
// with the correct environment for that pod.
scriptPath := path.Join(eventExecDir, executable.Service.Name)
file, err := os.OpenFile(scriptPath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0744)
defer file.Close()
if err != nil {
l.Logger.WithField("err", err).Errorln("Could not write to event dir path")
}
err = executable.WriteExecutor(file)
if err != nil {
l.Logger.WithField("err", err).Errorln("Could not install new hook")
}
}
// for convenience as we do with regular launchables, make these ones
// current under the launchable directory
err = launchable.MakeCurrent()
if err != nil {
l.Logger.WithField("err", err).Errorln("Could not update the current hook")
}
}
return nil
}
func main() {
kingpin.Version(version.VERSION)
kingpin.Parse()
log.Println("Starting bootstrap")
hostname, err := os.Hostname()
if err != nil {
log.Fatalf("error getting node name: %v", err)
}
nodeName := types.NodeName(hostname)
agentManifest, err := manifest.FromPath(*agentManifestPath)
if err != nil {
log.Fatalln("Could not get agent manifest: %s", err)
}
log.Println("Installing and launching consul")
podFactory := pods.NewFactory(*podRoot, nodeName)
var consulPod *pods.Pod
var consulManifest manifest.Manifest
if *existingConsul == "" {
consulManifest, err = manifest.FromPath(*consulManifestPath)
if err != nil {
log.Fatalf("Could not get consul manifest: %s", err)
}
// Consul will never have a uuid (for now)
consulPod = podFactory.NewLegacyPod(consulManifest.ID())
err = installConsul(consulPod, consulManifest, *registryURL)
if err != nil {
log.Fatalf("Could not install consul: %s", err)
}
} else {
log.Printf("Using existing Consul at %s\n", *existingConsul)
consulPod, err = pods.PodFromPodHome(nodeName, *existingConsul)
if err != nil {
log.Fatalf("The existing consul pod is invalid: %s", err)
}
consulManifest, err = consulPod.CurrentManifest()
if err != nil {
log.Fatalf("Cannot get the current consul manifest: %s", err)
}
}
if err = verifyConsulUp(*timeout); err != nil {
log.Fatalln(err)
}
time.Sleep(500 * time.Millisecond)
// schedule consul in the reality store as well, to ensure the preparers do
// not all restart their consul agents simultaneously after bootstrapping
err = scheduleForThisHost(consulManifest, true)
if err != nil {
log.Fatalf("Could not register consul in the intent store: %s", err)
}
log.Println("Registering base agent in consul")
err = scheduleForThisHost(agentManifest, false)
if err != nil {
log.Fatalf("Could not register base agent with consul: %s", err)
}
log.Println("Installing and launching base agent")
err = installBaseAgent(podFactory, agentManifest, *registryURL)
if err != nil {
log.Fatalf("Could not install base agent: %s", err)
}
if err := verifyReality(30*time.Second, consulManifest.ID(), agentManifest.ID()); err != nil {
log.Fatalln(err)
}
log.Println("Bootstrapping complete")
}
// Populates the given directory with executor scripts for each launch script of
// the given pod, which must be installed. Any orphaned executor scripts (from a
// past install, but no longer present in this pod) will be cleaned out.
func InstallHookScripts(dir string, hookPod *pods.Pod, manifest pods.Manifest, logger logging.Logger) error {
err := os.MkdirAll(dir, 0755)
if err != nil {
return err
}
// TODO: globbing based on the structure of the name is gross, each hook pod
// should have its own dir of scripts and running hooks should iterate over
// the directories
rmPattern := filepath.Join(dir, fmt.Sprintf("%s__*", hookPod.Id))
matches, err := filepath.Glob(rmPattern)
if err != nil {
// error return from filepath.Glob is guaranteed to be ErrBadPattern
return util.Errorf("error while removing old hook scripts: pattern %q is malformed", rmPattern)
}
for _, match := range matches {
err = os.Remove(match)
if err != nil {
logger.WithErrorAndFields(err, logrus.Fields{"script_path": match}).Errorln("Could not remove old hook script")
}
}
launchables, err := hookPod.Launchables(manifest)
if err != nil {
return err
}
for _, launchable := range launchables {
if launchable.Type() != "hoist" {
logger.WithFields(logrus.Fields{
"id": launchable.ID(),
"type": launchable.Type(),
}).Errorln("hook disabled: unsupported launchable type")
continue
}
executables, err := launchable.Executables(runit.DefaultBuilder)
if err != nil {
return err
}
for _, executable := range executables {
// Write a script to the event directory that executes the pod's executables
// with the correct environment for that pod.
scriptPath := filepath.Join(dir, executable.Service.Name)
file, err := os.OpenFile(scriptPath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0744)
defer file.Close()
if err != nil {
logger.WithErrorAndFields(err, logrus.Fields{"script_path": scriptPath}).Errorln("Could not open new hook script")
}
err = executable.WriteExecutor(file)
if err != nil {
logger.WithErrorAndFields(err, logrus.Fields{"script_path": scriptPath}).Errorln("Could not write new hook script")
}
}
// for convenience as we do with regular launchables, make these ones
// current under the launchable directory
err = launchable.MakeCurrent()
if err != nil {
logger.WithErrorAndFields(err, logrus.Fields{"launchable_id": launchable.ID()}).
Errorln("Could not set hook launchable to current")
}
}
return nil
}
func main() {
kingpin.Version(version.VERSION)
kingpin.Parse()
log.Println("Starting bootstrap")
agentManifest, err := pods.ManifestFromPath(*agentManifestPath)
if err != nil {
log.Fatalln("Could not get agent manifest: %s", err)
}
log.Println("Installing and launching consul")
var consulPod *pods.Pod
var consulManifest *pods.Manifest
if *existingConsul == "" {
consulManifest, err = pods.ManifestFromPath(*consulManifestPath)
if err != nil {
log.Fatalf("Could not get consul manifest: %s", err)
}
consulPod = pods.NewPod(consulManifest.ID(), pods.PodPath(*podRoot, consulManifest.ID()))
err = InstallConsul(consulPod, consulManifest)
if err != nil {
log.Fatalf("Could not install consul: %s", err)
}
} else {
log.Printf("Using existing Consul at %s\n", *existingConsul)
consulPod, err = pods.ExistingPod(*existingConsul)
if err != nil {
log.Fatalf("The existing consul pod is invalid: %s", err)
}
consulManifest, err = consulPod.CurrentManifest()
if err != nil {
log.Fatalf("Cannot get the current consul manifest: %s", err)
}
}
if err = VerifyConsulUp(*timeout); err != nil {
log.Fatalln(err)
}
time.Sleep(500 * time.Millisecond)
// schedule consul in the reality store as well, to ensure the preparers do
// not all restart their consul agents simultaneously after bootstrapping
err = ScheduleForThisHost(consulManifest, true)
if err != nil {
log.Fatalf("Could not register consul in the intent store: %s", err)
}
log.Println("Registering base agent in consul")
err = ScheduleForThisHost(agentManifest, false)
if err != nil {
log.Fatalf("Could not register base agent with consul: %s", err)
}
log.Println("Installing and launching base agent")
err = InstallBaseAgent(agentManifest)
if err != nil {
log.Fatalf("Could not install base agent: %s", err)
}
if err := VerifyReality(30*time.Second, consulManifest.ID(), agentManifest.ID()); err != nil {
log.Fatalln(err)
}
log.Println("Bootstrapping complete")
}
// no return value, no output channels. This should do everything it needs to do
// without outside intervention (other than being signalled to quit)
func (p *Preparer) handlePods(podChan <-chan ManifestPair, quit <-chan struct{}) {
// install new launchables
var nextLaunch ManifestPair
// used to track if we have work to do (i.e. pod manifest came through channel
// and we have yet to operate on it)
working := false
var manifestLogger logging.Logger
// The design of p2-preparer is to continuously retry installation
// failures, for example downloading of the launchable. An exponential
// backoff is important to avoid putting undue load on the artifact
// server, for example.
backoffTime := minimumBackoffTime
for {
select {
case <-quit:
return
case nextLaunch = <-podChan:
backoffTime = minimumBackoffTime
var sha string
// TODO: handle errors appropriately from SHA().
if nextLaunch.Intent != nil {
sha, _ = nextLaunch.Intent.SHA()
} else {
sha, _ = nextLaunch.Reality.SHA()
}
manifestLogger = p.Logger.SubLogger(logrus.Fields{
"pod": nextLaunch.ID,
"sha": sha,
"pod_unique_key": nextLaunch.PodUniqueKey,
})
manifestLogger.NoFields().Debugln("New manifest received")
working = true
case <-time.After(backoffTime):
if working {
var pod *pods.Pod
var err error
if nextLaunch.PodUniqueKey == "" {
pod = p.podFactory.NewLegacyPod(nextLaunch.ID)
} else {
pod, err = p.podFactory.NewUUIDPod(nextLaunch.ID, nextLaunch.PodUniqueKey)
if err != nil {
manifestLogger.WithError(err).Errorln("Could not initialize pod")
break
}
}
// TODO better solution: force the preparer to have a 0s default timeout, prevent KILLs
if pod.Id == constants.PreparerPodID {
pod.DefaultTimeout = time.Duration(0)
}
effectiveLogBridgeExec := p.logExec
// pods that are in the blacklist for this preparer shall not use the
// preparer's log exec. Instead, they will use the default svlogd logexec.
for _, podID := range p.logBridgeBlacklist {
if pod.Id.String() == podID {
effectiveLogBridgeExec = svlogdExec
break
}
}
pod.SetLogBridgeExec(effectiveLogBridgeExec)
pod.SetFinishExec(p.finishExec)
// podChan is being fed values gathered from a kp.Watch() in
// WatchForPodManifestsForNode(). If the watch returns a new pair of
// intent/reality values before the previous change has finished
// processing in resolvePair(), the reality value will be stale. This
// leads to a bug where the preparer will appear to update a package
// and when that is finished, "update" it again.
//
// Example ordering of bad events:
// 1) update to /intent for pod A comes in, /reality is read and
// resolvePair() handles it
// 2) before resolvePair() finishes, another /intent update comes in,
// and /reality is read but hasn't been changed. This update cannot
// be processed until the previous resolvePair() call finishes, and
// updates /reality. Now the reality value used here is stale. We
// want to refresh our /reality read so we don't restart the pod if
// intent didn't change between updates.
//
// The correct solution probably involves watching reality and intent
// and feeding updated pairs to a control loop.
//
// This is a quick fix to ensure that the reality value being used is
// up-to-date. The de-bouncing logic in this method should ensure that the
// intent value is fresh (to the extent that Consul is timely). Fetching
// the reality value again ensures its freshness too.
if nextLaunch.PodUniqueKey == "" {
// legacy pod, get reality manifest from reality tree
reality, _, err := p.store.Pod(kp.REALITY_TREE, p.node, nextLaunch.ID)
if err == pods.NoCurrentManifest {
nextLaunch.Reality = nil
} else if err != nil {
manifestLogger.WithError(err).Errorln("Error getting reality manifest")
break
} else {
nextLaunch.Reality = reality
}
} else {
// uuid pod, get reality manifest from pod status
status, _, err := p.podStatusStore.Get(nextLaunch.PodUniqueKey)
switch {
case err != nil && !statusstore.IsNoStatus(err):
manifestLogger.WithError(err).Errorln("Error getting reality manifest from pod status")
break
case statusstore.IsNoStatus(err):
nextLaunch.Reality = nil
default:
manifest, err := manifest.FromBytes([]byte(status.Manifest))
if err != nil {
manifestLogger.WithError(err).Errorln("Error parsing reality manifest from pod status")
break
}
nextLaunch.Reality = manifest
}
}
ok := p.resolvePair(nextLaunch, pod, manifestLogger)
if ok {
nextLaunch = ManifestPair{}
working = false
// Reset the backoff time
backoffTime = minimumBackoffTime
} else {
// Double the backoff time with a maximum of 1 minute
backoffTime = backoffTime * 2
if backoffTime > 1*time.Minute {
backoffTime = 1 * time.Minute
}
}
}
}
}
}
