func (h *HookEnv) Manifest() (manifest.Manifest, error) { path := os.Getenv(HOOKED_POD_MANIFEST_ENV_VAR) if path == "" { return nil, util.Errorf("No manifest exported") } return manifest.FromPath(path) }
func (r rctlParams) Create(manifestPath, nodeSelector string, podLabels map[string]string, rcLabels map[string]string) { manifest, err := manifest.FromPath(manifestPath) if err != nil { r.logger.WithErrorAndFields(err, logrus.Fields{ "manifest": manifestPath, }).Fatalln("Could not read pod manifest") } nodeSel, err := klabels.Parse(nodeSelector) if err != nil { r.logger.WithErrorAndFields(err, logrus.Fields{ "selector": nodeSelector, }).Fatalln("Could not parse node selector") } newRC, err := r.rcs.Create(manifest, nodeSel, klabels.Set(podLabels)) if err != nil { r.logger.WithError(err).Fatalln("Could not create replication controller in Consul") } r.logger.WithField("id", newRC.ID).Infoln("Created new replication controller") err = r.labeler.SetLabels(labels.RC, newRC.ID.String(), rcLabels) if err != nil { r.logger.WithError(err).Fatalln("Could not label replication controller") } }
func schedule(client client.Client, logger logging.Logger) { m, err := manifest.FromPath(*manifestFile) if err != nil { logger.Fatalf("Could not read manifest: %s", err) } podUniqueKey, err := client.Schedule(m, types.NodeName(*node)) if err != nil { logger.Fatalf("Could not schedule: %s", err) } output := struct { PodID types.PodID `json:"pod_id"` PodUniqueKey types.PodUniqueKey `json:"pod_unique_key"` }{ PodID: m.ID(), PodUniqueKey: podUniqueKey, } outBytes, err := json.Marshal(output) if err != nil { logger.Infof("Scheduled pod with key: %s", podUniqueKey) return } fmt.Println(string(outBytes)) }
func TestWriteManifestWillReturnOldManifestTempPath(t *testing.T) { existing := getTestPodManifest(t).GetBuilder() updated := getUpdatedManifest(t).GetBuilder() poddir, err := ioutil.TempDir("", "poddir") Assert(t).IsNil(err, "couldn't create tempdir") pod := newPodWithHome("testPod", poddir, "testNode") // set the RunAs user to the user running the test, because when we // write files we need an owner. currUser, err := user.Current() Assert(t).IsNil(err, "Could not get the current user") existing.SetRunAsUser(currUser.Username) updated.SetRunAsUser(currUser.Username) manifestContent, err := existing.GetManifest().Marshal() Assert(t).IsNil(err, "couldn't get manifest bytes") err = ioutil.WriteFile(pod.currentPodManifestPath(), manifestContent, 0744) Assert(t).IsNil(err, "should have written current manifest") oldPath, err := pod.WriteCurrentManifest(updated.GetManifest()) Assert(t).IsNil(err, "should have written the current manifest and linked the old one") writtenOld, err := manifest.FromPath(oldPath) Assert(t).IsNil(err, "should have written a manifest to the old path") manifestMustEqual(existing.GetManifest(), writtenOld, t) writtenCurrent, err := pod.CurrentManifest() Assert(t).IsNil(err, "the manifest was not written properly") manifestMustEqual(updated.GetManifest(), writtenCurrent, t) }
func userCreationHookManifest(tmpdir string) (manifest.Manifest, error) { createUserPath := path.Join(tmpdir, "create_user") script := `#!/usr/bin/env bash set -e mkdir -p $HOOKED_POD_HOME /sbin/adduser $HOOKED_POD_ID -d $HOOKED_POD_HOME ` err := ioutil.WriteFile(createUserPath, []byte(script), 0744) if err != nil { return nil, err } cmd := exec.Command("p2-bin2pod", "--work-dir", tmpdir, createUserPath) createUserBin2Pod, err := executeBin2Pod(cmd) if err != nil { return nil, err } if err = signBuild(createUserBin2Pod.TarPath); err != nil { return nil, err } manifestPath := createUserBin2Pod.ManifestPath userHookManifest, err := manifest.FromPath(manifestPath) if err != nil { return nil, err } builder := userHookManifest.GetBuilder() builder.SetRunAsUser("root") userHookManifest = builder.GetManifest() return userHookManifest, nil }
func (pod *Pod) CurrentManifest() (manifest.Manifest, error) { currentManPath := pod.currentPodManifestPath() if _, err := os.Stat(currentManPath); os.IsNotExist(err) { return nil, NoCurrentManifest } return manifest.FromPath(currentManPath) }
func main() { kingpin.Version(version.VERSION) _, opts, _ := flags.ParseWithConsulOptions() client := kp.NewConsulClient(opts) store := kp.NewConsulStore(client) podStore := podstore.NewConsul(client.KV()) if *nodeName == "" { hostname, err := os.Hostname() if err != nil { log.Fatalf("Could not get the hostname to do scheduling: %s", err) } *nodeName = hostname } if *manifestPath == "" { kingpin.Usage() log.Fatalln("No manifest given") } podManifest, err := manifest.FromPath(*manifestPath) if err != nil { log.Fatalf("Could not read manifest at %s: %s\n", *manifestPath, err) } out := schedule.Output{ PodID: podManifest.ID(), } if *uuidPod { out.PodUniqueKey, err = podStore.Schedule(podManifest, types.NodeName(*nodeName)) if err != nil { log.Fatalf("Could not schedule pod: %s", err) } } else { // Legacy pod podPrefix := kp.INTENT_TREE if *hookGlobal { podPrefix = kp.HOOK_TREE } _, err := store.SetPod(podPrefix, types.NodeName(*nodeName), podManifest) if err != nil { log.Fatalf("Could not write manifest %s to intent store: %s\n", podManifest.ID(), err) } } outBytes, err := json.Marshal(out) if err != nil { log.Fatalf("Successfully scheduled manifest but couldn't marshal JSON output") } fmt.Println(string(outBytes)) }
func scheduleUserCreationHook(tmpdir string) error { createUserPath := path.Join(tmpdir, "create_user") script := `#!/usr/bin/env bash set -e mkdir -p $HOOKED_POD_HOME /sbin/adduser $HOOKED_POD_ID -d $HOOKED_POD_HOME ` err := ioutil.WriteFile(createUserPath, []byte(script), 0744) if err != nil { return err } cmd := exec.Command("p2-bin2pod", "--work-dir", tmpdir, createUserPath) createUserBin2Pod, err := executeBin2Pod(cmd) if err != nil { return err } if err = signBuild(createUserBin2Pod.TarPath); err != nil { return err } manifestPath := createUserBin2Pod.ManifestPath userHookManifest, err := manifest.FromPath(manifestPath) if err != nil { return err } builder := userHookManifest.GetBuilder() builder.SetRunAsUser("root") userHookManifest = builder.GetManifest() contents, err := userHookManifest.Marshal() if err != nil { return err } ioutil.WriteFile(manifestPath, contents, 0644) manifestPath, err = signManifest(manifestPath, tmpdir) if err != nil { return err } return exec.Command("p2-schedule", "--hook", manifestPath).Run() }
func main() { kingpin.Version(version.VERSION) kingpin.Parse() if *nodeName == "" { hostname, err := os.Hostname() if err != nil { log.Fatalf("error getting node name: %v", err) } *nodeName = hostname } dir := hooks.Hooks(*hookDir, &logging.DefaultLogger) hookType, err := hooks.AsHookType(*hookType) if err != nil { log.Fatalln(err) } pod, err := pods.PodFromPodHome(types.NodeName(*nodeName), *podDir) if err != nil { log.Fatalln(err) } var podManifest manifest.Manifest if *manifestPath != "" { podManifest, err = manifest.FromPath(*manifestPath) if err != nil { log.Fatalln(err) } } else { podManifest, err = pod.CurrentManifest() if err != nil { log.Fatalln(err) } } log.Printf("About to run %s hooks for pod %s\n", hookType, pod.Home()) err = dir.RunHookType(hookType, pod, podManifest) if err != nil { log.Fatalln(err) } }
func main() { cmd, consulOpts := flags.ParseWithConsulOptions() client := kp.NewConsulClient(consulOpts) logger := logging.NewLogger(logrus.Fields{}) dsstore := dsstore.NewConsul(client, 3, &logger) applicator := labels.NewConsulApplicator(client, 3) switch cmd { case CmdCreate: minHealth, err := strconv.Atoi(*createMinHealth) if err != nil { log.Fatalf("Invalid value for minimum health, expected integer: %v", err) } name := ds_fields.ClusterName(*createName) manifest, err := manifest.FromPath(*createManifest) if err != nil { log.Fatalf("%s", err) } podID := manifest.ID() if *createTimeout <= time.Duration(0) { log.Fatalf("Timeout must be a positive non-zero value, got '%v'", *createTimeout) } selectorString := *createSelector if *createEverywhere { selectorString = klabels.Everything().String() } else if selectorString == "" { selectorString = klabels.Nothing().String() log.Fatal("Explicit everything selector not allowed, please use the --everwhere flag") } selector, err := parseNodeSelectorWithPrompt(klabels.Nothing(), selectorString, applicator) if err != nil { log.Fatalf("Error occurred: %v", err) } if err = confirmMinheathForSelector(minHealth, selector, applicator); err != nil { log.Fatalf("Error occurred: %v", err) } ds, err := dsstore.Create(manifest, minHealth, name, selector, podID, *createTimeout) if err != nil { log.Fatalf("err: %v", err) } fmt.Printf("%v has been created in consul", ds.ID) fmt.Println() case CmdGet: id := ds_fields.ID(*getID) ds, _, err := dsstore.Get(id) if err != nil { log.Fatalf("err: %v", err) } bytes, err := json.Marshal(ds) if err != nil { logger.WithError(err).Fatalln("Unable to marshal daemon set as JSON") } fmt.Printf("%s", bytes) case CmdList: dsList, err := dsstore.List() if err != nil { log.Fatalf("err: %v", err) } podID := types.PodID(*listPod) for _, ds := range dsList { if *listPod == "" || podID == ds.PodID { fmt.Printf("%s/%s:%s\n", ds.PodID, ds.Name, ds.ID) } } case CmdEnable: id := ds_fields.ID(*enableID) mutator := func(ds ds_fields.DaemonSet) (ds_fields.DaemonSet, error) { if !ds.Disabled { return ds, util.Errorf("Daemon set has already been enabled") } ds.Disabled = false return ds, nil } _, err := dsstore.MutateDS(id, mutator) if err != nil { log.Fatalf("err: %v", err) } fmt.Printf("The daemon set '%s' has been successfully enabled in consul", id.String()) fmt.Println() case CmdDisable: id := ds_fields.ID(*disableID) mutator := func(ds ds_fields.DaemonSet) (ds_fields.DaemonSet, error) { if ds.Disabled { return ds, util.Errorf("Daemon set has already been disabled") } ds.Disabled = true return ds, nil } _, err := dsstore.MutateDS(id, mutator) if err != nil { log.Fatalf("err: %v", err) } fmt.Printf("The daemon set '%s' has been successfully disabled in consul", id.String()) fmt.Println() case CmdDelete: id := ds_fields.ID(*deleteID) err := dsstore.Delete(id) if err != nil { log.Fatalf("err: %v", err) } fmt.Printf("The daemon set '%s' has been successfully deleted from consul", id.String()) fmt.Println() case CmdUpdate: id := ds_fields.ID(*updateID) mutator := func(ds ds_fields.DaemonSet) (ds_fields.DaemonSet, error) { changed := false if *updateMinHealth != "" { minHealth, err := strconv.Atoi(*updateMinHealth) if err != nil { log.Fatalf("Invalid value for minimum health, expected integer") } if ds.MinHealth != minHealth { changed = true ds.MinHealth = minHealth } } if *updateName != "" { name := ds_fields.ClusterName(*updateName) if ds.Name != name { changed = true ds.Name = name } } if *updateTimeout != TimeoutNotSpecified { if *updateTimeout <= time.Duration(0) { return ds, util.Errorf("Timeout must be a positive non-zero value, got '%v'", *createTimeout) } if ds.Timeout != *updateTimeout { changed = true ds.Timeout = *updateTimeout } } if *updateManifest != "" { manifest, err := manifest.FromPath(*updateManifest) if err != nil { return ds, util.Errorf("%s", err) } if manifest.ID() != ds.PodID { return ds, util.Errorf("Manifest ID of %s does not match daemon set's pod ID (%s)", manifest.ID(), ds.PodID) } dsSHA, err := ds.Manifest.SHA() if err != nil { return ds, util.Errorf("Unable to get SHA from consul daemon set manifest: %v", err) } newSHA, err := manifest.SHA() if err != nil { return ds, util.Errorf("Unable to get SHA from new manifest: %v", err) } if dsSHA != newSHA { changed = true ds.Manifest = manifest } } if updateSelectorGiven { selectorString := *updateSelector if *updateEverywhere { selectorString = klabels.Everything().String() } else if selectorString == "" { return ds, util.Errorf("Explicit everything selector not allowed, please use the --everwhere flag") } selector, err := parseNodeSelectorWithPrompt(ds.NodeSelector, selectorString, applicator) if err != nil { return ds, util.Errorf("Error occurred: %v", err) } if ds.NodeSelector.String() != selector.String() { changed = true ds.NodeSelector = selector } } if !changed { return ds, util.Errorf("No changes were made") } if updateSelectorGiven || *updateMinHealth != "" { if err := confirmMinheathForSelector(ds.MinHealth, ds.NodeSelector, applicator); err != nil { return ds, util.Errorf("Error occurred: %v", err) } } return ds, nil } _, err := dsstore.MutateDS(id, mutator) if err != nil { log.Fatalf("err: %v", err) } fmt.Printf("The daemon set '%s' has been successfully updated in consul", id.String()) fmt.Println() case CmdTestSelector: selectorString := *testSelectorString if *testSelectorEverywhere { selectorString = klabels.Everything().String() } else if selectorString == "" { fmt.Println("Explicit everything selector not allowed, please use the --everwhere flag") } selector, err := parseNodeSelector(selectorString) if err != nil { log.Fatalf("Error occurred: %v", err) } matches, err := applicator.GetMatches(selector, labels.NODE, false) if err != nil { log.Fatalf("Error getting matching labels: %v", err) } fmt.Println(matches) default: log.Fatalf("Unrecognized command %v", cmd) } }
func main() { kingpin.Version(version.VERSION) kingpin.Parse() log.Println("Starting bootstrap") hostname, err := os.Hostname() if err != nil { log.Fatalf("error getting node name: %v", err) } nodeName := types.NodeName(hostname) agentManifest, err := manifest.FromPath(*agentManifestPath) if err != nil { log.Fatalln("Could not get agent manifest: %s", err) } log.Println("Installing and launching consul") podFactory := pods.NewFactory(*podRoot, nodeName) var consulPod *pods.Pod var consulManifest manifest.Manifest if *existingConsul == "" { consulManifest, err = manifest.FromPath(*consulManifestPath) if err != nil { log.Fatalf("Could not get consul manifest: %s", err) } // Consul will never have a uuid (for now) consulPod = podFactory.NewLegacyPod(consulManifest.ID()) err = installConsul(consulPod, consulManifest, *registryURL) if err != nil { log.Fatalf("Could not install consul: %s", err) } } else { log.Printf("Using existing Consul at %s\n", *existingConsul) consulPod, err = pods.PodFromPodHome(nodeName, *existingConsul) if err != nil { log.Fatalf("The existing consul pod is invalid: %s", err) } consulManifest, err = consulPod.CurrentManifest() if err != nil { log.Fatalf("Cannot get the current consul manifest: %s", err) } } if err = verifyConsulUp(*timeout); err != nil { log.Fatalln(err) } time.Sleep(500 * time.Millisecond) // schedule consul in the reality store as well, to ensure the preparers do // not all restart their consul agents simultaneously after bootstrapping err = scheduleForThisHost(consulManifest, true) if err != nil { log.Fatalf("Could not register consul in the intent store: %s", err) } log.Println("Registering base agent in consul") err = scheduleForThisHost(agentManifest, false) if err != nil { log.Fatalf("Could not register base agent with consul: %s", err) } log.Println("Installing and launching base agent") err = installBaseAgent(podFactory, agentManifest, *registryURL) if err != nil { log.Fatalf("Could not install base agent: %s", err) } if err := verifyReality(30*time.Second, consulManifest.ID(), agentManifest.ID()); err != nil { log.Fatalln(err) } log.Println("Bootstrapping complete") }
func getUpdatedManifest(t *testing.T) manifest.Manifest { podPath := util.From(runtime.Caller(0)).ExpandPath("updated_manifest.yaml") pod, err := manifest.FromPath(podPath) Assert(t).IsNil(err, "couldn't read test manifest") return pod }
func generatePreparerPod(workdir string, userHookManifest manifest.Manifest) (string, error) { // build the artifact from HEAD output, err := exec.Command("go", "build", "github.com/square/p2/bin/p2-preparer").CombinedOutput() if err != nil { return "", util.Errorf("Couldn't build preparer: %s\nOutput:\n%s", err, string(output)) } wd, _ := os.Getwd() hostname, err := os.Hostname() if err != nil { return "", util.Errorf("Couldn't get hostname: %s", err) } // the test number forces the pod manifest to change every test run. testNumber := fmt.Sprintf("test=%d", rand.Intn(2000000000)) cmd := exec.Command("p2-bin2pod", "--work-dir", workdir, "--id", "p2-preparer", "--config", fmt.Sprintf("node_name=%s", hostname), "--config", testNumber, wd+"/p2-preparer") prepBin2Pod, err := executeBin2Pod(cmd) if err != nil { return "", err } if err = signBuild(prepBin2Pod.TarPath); err != nil { return "", err } manifest, err := manifest.FromPath(prepBin2Pod.ManifestPath) if err != nil { return "", err } builder := manifest.GetBuilder() builder.SetID("p2-preparer") envExtractorPath, err := exec.Command("which", "p2-finish-env-extractor").CombinedOutput() if err != nil { return "", fmt.Errorf("Could not find p2-finish-env-extractor on PATH") } userCreationHookBytes, err := userHookManifest.Marshal() if err != nil { return "", util.Errorf("Couldn't marshal user hook manifest: %s", err) } err = builder.SetConfig(map[interface{}]interface{}{ "preparer": map[interface{}]interface{}{ "auth": map[string]string{ "type": "keyring", "keyring": util.From(runtime.Caller(0)).ExpandPath("pubring.gpg"), }, "artifact_auth": map[interface{}]interface{}{ "type": "build", "keyring": util.From(runtime.Caller(0)).ExpandPath("pubring.gpg"), }, "ca_file": filepath.Join(certpath, "cert.pem"), "cert_file": filepath.Join(certpath, "cert.pem"), "key_file": filepath.Join(certpath, "key.pem"), "status_port": preparerStatusPort, "process_result_reporter_config": map[string]string{ "sqlite_database_path": sqliteFinishDatabasePath, "environment_extractor_path": strings.TrimSpace(string(envExtractorPath)), "workspace_dir_path": "/data/pods/p2-preparer/tmp", }, "hooks_manifest": string(userCreationHookBytes), }, }) if err != nil { return "", err } builder.SetRunAsUser("root") builder.SetStatusPort(preparerStatusPort) builder.SetStatusHTTP(true) manifest = builder.GetManifest() manifestBytes, err := manifest.Marshal() if err != nil { return "", err } err = ioutil.WriteFile(prepBin2Pod.ManifestPath, manifestBytes, 0644) if err != nil { return "", err } return prepBin2Pod.ManifestPath, err }
func generatePreparerPod(workdir string) (string, error) { // build the artifact from HEAD err := exec.Command("go", "build", "github.com/square/p2/bin/p2-preparer").Run() if err != nil { return "", util.Errorf("Couldn't build preparer: %s", err) } wd, _ := os.Getwd() hostname, err := os.Hostname() if err != nil { return "", util.Errorf("Couldn't get hostname: %s", err) } // the test number forces the pod manifest to change every test run. testNumber := fmt.Sprintf("test=%d", rand.Intn(2000000000)) cmd := exec.Command("p2-bin2pod", "--work-dir", workdir, "--id", "p2-preparer", "--config", fmt.Sprintf("node_name=%s", hostname), "--config", testNumber, wd+"/p2-preparer") prepBin2Pod, err := executeBin2Pod(cmd) if err != nil { return "", err } if err = signBuild(prepBin2Pod.TarPath); err != nil { return "", err } manifest, err := manifest.FromPath(prepBin2Pod.ManifestPath) if err != nil { return "", err } builder := manifest.GetBuilder() builder.SetID("p2-preparer") err = builder.SetConfig(map[interface{}]interface{}{ "preparer": map[interface{}]interface{}{ "auth": map[string]string{ "type": "keyring", "keyring": util.From(runtime.Caller(0)).ExpandPath("pubring.gpg"), }, "artifact_auth": map[interface{}]interface{}{ "type": "build", "keyring": util.From(runtime.Caller(0)).ExpandPath("pubring.gpg"), }, "ca_file": filepath.Join(certpath, "cert.pem"), "cert_file": filepath.Join(certpath, "cert.pem"), "key_file": filepath.Join(certpath, "key.pem"), "status_port": preparerStatusPort, }, }) if err != nil { return "", err } builder.SetRunAsUser("root") builder.SetStatusPort(preparerStatusPort) builder.SetStatusHTTP(true) manifest = builder.GetManifest() manifestBytes, err := manifest.Marshal() if err != nil { return "", err } err = ioutil.WriteFile(prepBin2Pod.ManifestPath, manifestBytes, 0644) if err != nil { return "", err } return prepBin2Pod.ManifestPath, err }