func (e EnvironmentExtractor) constructFinishFromEnvironment(exitCode int, exitStatus int) (FinishOutput, error) {
podID := os.Getenv(pods.PodIDEnvVar)
if podID == "" {
return FinishOutput{}, util.Errorf("No %s env var set", pods.PodIDEnvVar)
}
launchableID := os.Getenv(pods.LaunchableIDEnvVar)
if launchableID == "" {
return FinishOutput{}, util.Errorf("No %s env var set", pods.LaunchableIDEnvVar)
}
entryPoint := os.Getenv(launch.EntryPointEnvVar)
if entryPoint == "" {
return FinishOutput{}, util.Errorf("No %s env var set", launch.EntryPointEnvVar)
}
// It's okay if this one is missing, most pods are "legacy" pods that have a blank unique key
podUniqueKey := os.Getenv(pods.PodUniqueKeyEnvVar)
return FinishOutput{
PodID: types.PodID(podID),
LaunchableID: launch.LaunchableID(launchableID),
EntryPoint: entryPoint,
PodUniqueKey: types.PodUniqueKey(podUniqueKey),
ExitCode: exitCode,
ExitStatus: exitStatus,
}, nil
}
func TestLabelsOnCreate(t *testing.T) {
store := consulStoreWithFakeKV()
podID := types.PodID("pod_id")
az := fields.AvailabilityZone("us-west")
clusterName := fields.ClusterName("cluster_name")
selector := klabels.Everything().
Add(fields.PodIDLabel, klabels.EqualsOperator, []string{podID.String()}).
Add(fields.AvailabilityZoneLabel, klabels.EqualsOperator, []string{az.String()}).
Add(fields.ClusterNameLabel, klabels.EqualsOperator, []string{clusterName.String()})
annotations := fields.Annotations(map[string]interface{}{
"foo": "bar",
})
pc, err := store.Create(podID, az, clusterName, selector, annotations, kptest.NewSession())
if err != nil {
t.Fatalf("Unable to create pod cluster: %s", err)
}
matches, err := store.applicator.GetMatches(selector, labels.PC)
if err != nil {
t.Fatalf("Unable to check for label match on new pod cluster: %s", err)
}
if len(matches) != 1 {
t.Errorf("Expected one pod cluster to match label selector")
}
if fields.ID(matches[0].ID) != pc.ID {
t.Errorf("The pod cluster selector didn't match the new pod cluster")
}
}
func TestStartWatchBasic(t *testing.T) {
quitCh := make(chan struct{})
defer close(quitCh)
hs, checker := NewFakeHealthStore()
go hs.StartWatch(quitCh)
node := types.NodeName("abc01.sjc1")
podID1 := types.PodID("podID1")
podID2 := types.PodID("podID2")
result := hs.Fetch(podID1, node)
if result != nil {
t.Errorf("expected cache to start empty, found %v", result)
}
checker.Send([]*health.Result{
{ID: podID1, Node: node},
{ID: podID2, Node: node},
})
result = hs.Fetch(podID1, node)
if result == nil {
t.Errorf("expected health store to have %s", podID1)
}
result = hs.Fetch(podID2, node)
if result == nil {
t.Errorf("expected health store to have %s", podID2)
}
}
func TestList(t *testing.T) {
store := consulStoreWithFakeKV()
// Create first DaemonSet
firstPodID := types.PodID("some_pod_id")
minHealth := 0
clusterName := ds_fields.ClusterName("some_name")
azLabel := pc_fields.AvailabilityZone("some_zone")
selector := klabels.Everything().
Add(pc_fields.AvailabilityZoneLabel, klabels.EqualsOperator, []string{azLabel.String()})
manifestBuilder := manifest.NewBuilder()
manifestBuilder.SetID(firstPodID)
firstManifest := manifestBuilder.GetManifest()
timeout := replication.NoTimeout
firstDS, err := store.Create(firstManifest, minHealth, clusterName, selector, firstPodID, timeout)
if err != nil {
t.Fatalf("Unable to create daemon set: %s", err)
}
// Create second DaemonSet
secondPodID := types.PodID("different_pod_id")
manifestBuilder = manifest.NewBuilder()
manifestBuilder.SetID(secondPodID)
secondManifest := manifestBuilder.GetManifest()
secondDS, err := store.Create(secondManifest, minHealth, clusterName, selector, secondPodID, timeout)
if err != nil {
t.Fatalf("Unable to create daemon set: %s", err)
}
daemonSetList, err := store.List()
if err != nil {
t.Fatalf("Error getting list of daemon sets: %s", err)
}
Assert(t).AreEqual(len(daemonSetList), 2, "Unexpected number of daemon sets listed")
for _, daemonSet := range daemonSetList {
if daemonSet.ID == firstDS.ID {
Assert(t).AreEqual(daemonSet.PodID, firstPodID, "Listed daemon set pod ids were not equal")
} else if daemonSet.PodID == secondDS.PodID {
Assert(t).AreEqual(daemonSet.PodID, secondPodID, "Listed daemon set pod ids were not equal")
} else {
t.Errorf("Unexpected daemon set listed: %v", daemonSet)
}
}
}
func (rc *replicationController) ensureConsistency(current types.PodLocations) error {
manifestSHA, err := rc.Manifest.SHA()
if err != nil {
return err
}
for _, pod := range current {
intent, _, err := rc.kpStore.Pod(kp.INTENT_TREE, pod.Node, types.PodID(pod.PodID))
if err != nil && err != pods.NoCurrentManifest {
return err
}
var intentSHA string
if intent != nil {
intentSHA, err = intent.SHA()
if err != nil {
rc.logger.WithError(err).WithField("node", pod.Node).Warn("Could not hash manifest to determine consistency of intent")
}
if intentSHA == manifestSHA {
continue
}
}
rc.logger.WithField("node", pod.Node).WithField("intentManifestSHA", intentSHA).Info("Found inconsistency in scheduled manifest")
if err := rc.schedule(pod.Node); err != nil {
return err
}
}
return nil
}
// UpdatePods looks at the pods currently being monitored and
// compares that to what the reality store indicates should be
// running. UpdatePods then shuts down the monitors for dead
// pods and creates PodWatch structs for new pods.
func TestUpdatePods(t *testing.T) {
var current []PodWatch
var reality []kp.ManifestResult
// ids for current: 0, 1, 2, 3
for i := 0; i < 4; i++ {
current = append(current, *newWatch(types.PodID(strconv.Itoa(i))))
}
// ids for reality: 1, 2, test
for i := 1; i < 3; i++ {
// Health checking is not supported for uuid pods, so ensure that even
// if /reality contains a uuid pod we don't actually watch its health
uuidKeyResult := newManifestResult("some_uuid_pod")
uuidKeyResult.PodUniqueKey = types.NewPodUUID()
reality = append(reality, uuidKeyResult)
reality = append(reality, newManifestResult(current[i].manifest.ID()))
}
reality = append(reality, newManifestResult("test"))
// ids for pods: 1, 2, test
// 0, 3 should have values in their shutdownCh
logger := logging.NewLogger(logrus.Fields{})
pods := updatePods(&MockHealthManager{}, nil, nil, current, reality, "", &logger)
Assert(t).AreEqual(true, <-current[0].shutdownCh, "this PodWatch should have been shutdown")
Assert(t).AreEqual(true, <-current[3].shutdownCh, "this PodWatch should have been shutdown")
Assert(t).AreEqual(current[1].manifest.ID(), pods[0].manifest.ID(), "pod with id:1 should have been returned")
Assert(t).AreEqual(current[2].manifest.ID(), pods[1].manifest.ID(), "pod with id:1 should have been returned")
Assert(t).AreEqual("test", string(pods[2].manifest.ID()), "should have added pod with id:test to list")
}
func TestCreate(t *testing.T) {
testAZ := fields.AvailabilityZone("west-coast")
testCN := fields.ClusterName("test")
testPodID := types.PodID("pod")
selector := labels.Everything().
Add(fields.PodIDLabel, labels.EqualsOperator, []string{testPodID.String()}).
Add(fields.AvailabilityZoneLabel, labels.EqualsOperator, []string{testAZ.String()}).
Add(fields.ClusterNameLabel, labels.EqualsOperator, []string{testCN.String()})
session := kptest.NewSession()
pcstore := pcstoretest.NewFake()
pcController := NewPodCluster(testAZ, testCN, testPodID, pcstore, selector, session)
annotations := map[string]string{
"load_balancer_info": "totally",
"pager_information": "555-111-2222",
}
buf, err := json.Marshal(annotations)
if err != nil {
t.Errorf("json marshal error: %v", err)
}
var testAnnotations fields.Annotations
if err := json.Unmarshal(buf, &testAnnotations); err != nil {
t.Errorf("json unmarshal error: %v", err)
}
pc, err := pcController.Create(fields.Annotations(testAnnotations))
if err != nil {
t.Errorf("got error during creation: %v", err)
}
if pc.ID == "" {
t.Error("got empty pc ID")
}
if pc.PodID != testPodID {
t.Errorf("Expected to get %s, got: %v", pc.PodID, testPodID)
}
if pc.Name != testCN {
t.Errorf("Expected to get %s, got: %v", testCN, pc.Name)
}
if pc.AvailabilityZone != testAZ {
t.Errorf("Expected to get %s, got: %v", testAZ, pc.AvailabilityZone)
}
if pc.PodSelector.String() != selector.String() {
t.Errorf("Expected to get %s, got: %v", selector, pc.PodSelector)
}
if pc.Annotations["load_balancer_info"] != testAnnotations["load_balancer_info"] {
t.Errorf("Expected to get %s, got: %v", testAnnotations, pc.Annotations)
}
if pc.Annotations["pager_information"] != testAnnotations["pager_information"] {
t.Errorf("Expected to get %s, got: %v", testAnnotations, pc.Annotations)
}
}
func (h *HookEnv) PodID() (types.PodID, error) {
id := os.Getenv(HOOKED_POD_ID_ENV_VAR)
if id == "" {
return "", util.Errorf("Did not provide a pod ID to use")
}
return types.PodID(id), nil
}
func (rm *P2RM) deleteLegacyPod() error {
_, err := rm.Store.DeletePod(kp.INTENT_TREE, rm.NodeName, types.PodID(rm.PodID))
if err != nil {
return fmt.Errorf("unable to remove pod: %v", err)
}
return nil
}
func handlePodRemoval(consulClient consulutil.ConsulClient, labeler labels.ApplicatorWithoutWatches) error {
var rm *P2RM
if *podUniqueKey != "" {
rm = NewUUIDP2RM(consulClient, types.PodUniqueKey(*podUniqueKey), types.PodID(*podName), labeler)
} else {
if *nodeName == "" {
hostname, err := os.Hostname()
if err != nil {
return fmt.Errorf("error getting hostname. use --node to specify a node: %v\n", err)
}
*nodeName = hostname
}
rm = NewLegacyP2RM(consulClient, types.PodID(*podName), types.NodeName(*nodeName), labeler)
}
podIsManagedByRC, rcID, err := rm.checkForManagingReplicationController()
if err != nil {
return err
}
if !podIsManagedByRC {
err = rm.deletePod()
if err != nil {
return err
}
}
if podIsManagedByRC && !*deallocation {
return fmt.Errorf("error: %s is managed by replication controller: %s\n"+
"It's possible you meant you deallocate this pod on this node. If so, please confirm your intention with --deallocate\n", *nodeName, rcID)
}
if podIsManagedByRC && *deallocation {
err = rm.decrementDesiredCount(rcID)
if err != nil {
return fmt.Errorf("Encountered error deallocating from the RC %s. You may attempt this command again or use `p2-rctl` to cleanup manually.\n%v",
rcID,
err)
}
}
fmt.Printf("%s: successfully removed %s\n", rm.NodeName, rm.PodID)
return nil
}
func TestWatch(t *testing.T) {
store := consulStoreWithFakeKV()
podID := types.PodID("pod_id")
az := fields.AvailabilityZone("us-west")
clusterName := fields.ClusterName("cluster_name")
selector := klabels.Everything().
Add(fields.PodIDLabel, klabels.EqualsOperator, []string{podID.String()}).
Add(fields.AvailabilityZoneLabel, klabels.EqualsOperator, []string{az.String()}).
Add(fields.ClusterNameLabel, klabels.EqualsOperator, []string{clusterName.String()})
annotations := fields.Annotations(map[string]interface{}{
"foo": "bar",
})
var watched WatchedPodClusters
session := kptest.NewSession()
pc, err := store.Create(podID, az, clusterName, selector, annotations, session)
if err != nil {
t.Fatalf("Unable to create first pod cluster: %s", err)
}
pc2, err := store.Create(podID, "us-east", clusterName, selector, annotations, session)
if err != nil {
t.Fatalf("Unable to create second pod cluster: %s", err)
}
quit := make(chan struct{})
defer close(quit)
watch := store.Watch(quit)
select {
case watchedPC := <-watch:
watched = watchedPC
case <-time.After(5 * time.Second):
t.Fatal("nothing on the channel")
}
if len(watched.Clusters) != 2 {
t.Fatalf("Expected to get two watched PodClusters, but did not: got %v", len(watched.Clusters))
}
expectedIDs := []fields.ID{pc.ID, pc2.ID}
for _, id := range expectedIDs {
found := false
for _, pc := range watched.Clusters {
if id == pc.ID {
found = true
break
}
}
if !found {
t.Errorf("Expected to find id '%s' among watch results, but was not present", id)
}
}
}
func (h AlwaysHappyHealthChecker) Service(serviceID string) (map[types.NodeName]health.Result, error) {
results := make(map[types.NodeName]health.Result)
for _, node := range h.allNodes {
results[node] = health.Result{
ID: types.PodID(serviceID),
Status: health.Passing,
}
}
return results, nil
}
func createPodCluster(store *consulStore, t *testing.T) fields.PodCluster {
podID := types.PodID("pod_id")
az := fields.AvailabilityZone("us-west")
clusterName := fields.ClusterName("cluster_name")
selector := klabels.Everything().
Add(fields.PodIDLabel, klabels.EqualsOperator, []string{podID.String()}).
Add(fields.AvailabilityZoneLabel, klabels.EqualsOperator, []string{az.String()}).
Add(fields.ClusterNameLabel, klabels.EqualsOperator, []string{clusterName.String()})
annotations := fields.Annotations(map[string]interface{}{
"foo": "bar",
})
session := kptest.NewSession()
pc, err := store.Create(podID, az, clusterName, selector, annotations, session)
if err != nil {
t.Fatalf("Unable to create pod cluster: %s", err)
}
if pc.ID == "" {
t.Errorf("pod cluster should have an id")
}
if pc.PodID == "" {
t.Errorf("pod cluster should have a pod id")
}
if pc.PodID != podID {
t.Errorf("pod id wasn't properly set. Wanted '%s' got '%s'", podID, pc.PodID)
}
if pc.AvailabilityZone != az {
t.Errorf("availability zone wasn't properly set. Wanted '%s' got '%s'", az, pc.AvailabilityZone)
}
if pc.Name != clusterName {
t.Errorf("cluster name wasn't properly set. Wanted '%s' got '%s'", clusterName, pc.Name)
}
testLabels := klabels.Set{
fields.PodIDLabel: podID.String(),
fields.AvailabilityZoneLabel: az.String(),
fields.ClusterNameLabel: clusterName.String(),
}
if matches := pc.PodSelector.Matches(testLabels); !matches {
t.Errorf("the pod cluster has a bad pod selector")
}
if pc.Annotations["foo"] != "bar" {
t.Errorf("Annotations didn't match expected")
}
return pc
}
func NodeAndPodIDFromPodLabel(labeled Labeled) (types.NodeName, types.PodID, error) {
if labeled.LabelType != POD {
return "", "", util.Errorf("Label was not a pod label, was %s", labeled.LabelType)
}
parts := strings.SplitN(labeled.ID, "/", 2)
if len(parts) < 2 {
return "", "", util.Errorf("malformed pod label %s", labeled.ID)
}
return types.NodeName(parts[0]), types.PodID(parts[1]), nil
}
func createDaemonSet(store *consulStore, t *testing.T) ds_fields.DaemonSet {
podID := types.PodID("some_pod_id")
minHealth := 0
clusterName := ds_fields.ClusterName("some_name")
azLabel := pc_fields.AvailabilityZone("some_zone")
selector := klabels.Everything().
Add(pc_fields.AvailabilityZoneLabel, klabels.EqualsOperator, []string{azLabel.String()})
manifestBuilder := manifest.NewBuilder()
manifestBuilder.SetID(podID)
manifest := manifestBuilder.GetManifest()
timeout := replication.NoTimeout
ds, err := store.Create(manifest, minHealth, clusterName, selector, podID, timeout)
if err != nil {
t.Fatalf("Unable to create daemon set: %s", err)
}
if ds.ID == "" {
t.Error("daemon set should have an id")
}
Assert(t).AreNotEqual(ds.ID, "", "Daemon set should have an id")
Assert(t).AreNotEqual(ds.PodID, "", "Daemon set should have a pod id")
Assert(t).AreEqual(ds.PodID, podID, "Daemon set pod id was not set correctly")
Assert(t).AreEqual(ds.MinHealth, minHealth, "Daemon set minimum health was not set correctly")
Assert(t).AreEqual(ds.Name, clusterName, "Daemon set cluster name was not set correctly")
Assert(t).IsFalse(ds.Disabled, "Daemon set disabled field was not set correctly")
testLabels := klabels.Set{
pc_fields.AvailabilityZoneLabel: azLabel.String(),
}
if matches := ds.NodeSelector.Matches(testLabels); !matches {
t.Error("The daemon set has a bad node selector")
}
originalSHA, err := manifest.SHA()
if err != nil {
t.Fatal("Unable to retrieve SHA from manifest")
}
getSHA, err := ds.Manifest.SHA()
if err != nil {
t.Fatal("Unable to retrieve SHA from manifest retrieved from daemon set")
}
Assert(t).AreEqual(originalSHA, getSHA, "Daemon set manifest not set correctly")
return ds
}
func examplePodCluster() fields.PodCluster {
podId := "slug"
availabilityZone := "us-west"
clusterName := "production"
return fields.PodCluster{
PodID: types.PodID(podId),
AvailabilityZone: fields.AvailabilityZone(availabilityZone),
Name: fields.ClusterName(clusterName),
PodSelector: klabels.Everything(),
Annotations: fields.Annotations{},
}
}
func TestCreate(t *testing.T) {
store := consulStoreWithFakeKV()
createDaemonSet(store, t)
// Create a bad DaemonSet
podID := types.PodID("")
minHealth := 0
clusterName := ds_fields.ClusterName("some_name")
azLabel := pc_fields.AvailabilityZone("some_zone")
selector := klabels.Everything().
Add(pc_fields.AvailabilityZoneLabel, klabels.EqualsOperator, []string{azLabel.String()})
manifestBuilder := manifest.NewBuilder()
manifestBuilder.SetID("")
podManifest := manifestBuilder.GetManifest()
timeout := replication.NoTimeout
if _, err := store.Create(podManifest, minHealth, clusterName, selector, podID, timeout); err == nil {
t.Error("Expected create to fail on bad pod id")
}
podID = types.PodID("pod_id")
if _, err := store.Create(podManifest, minHealth, clusterName, selector, podID, timeout); err == nil {
t.Error("Expected create to fail on bad manifest pod id")
}
manifestBuilder = manifest.NewBuilder()
manifestBuilder.SetID("different_pod_id")
podManifest = manifestBuilder.GetManifest()
if _, err := store.Create(podManifest, minHealth, clusterName, selector, podID, timeout); err == nil {
t.Error("Expected create to fail on pod id and manifest pod id mismatch")
}
}
func TestDelete(t *testing.T) {
store := consulStoreWithFakeKV()
podID := types.PodID("pod_id")
az := fields.AvailabilityZone("us-west")
clusterName := fields.ClusterName("cluster_name")
selector := klabels.Everything().
Add(fields.PodIDLabel, klabels.EqualsOperator, []string{podID.String()}).
Add(fields.AvailabilityZoneLabel, klabels.EqualsOperator, []string{az.String()}).
Add(fields.ClusterNameLabel, klabels.EqualsOperator, []string{clusterName.String()})
annotations := fields.Annotations(map[string]interface{}{
"foo": "bar",
})
// Create a pod cluster
pc, err := store.Create(podID, az, clusterName, selector, annotations, kptest.NewSession())
if err != nil {
t.Fatalf("Unable to create pod cluster: %s", err)
}
pc, err = store.Get(pc.ID)
if err != nil {
t.Fatalf("Unable to get pod cluster: %s", err)
}
err = store.Delete(pc.ID)
if err != nil {
t.Fatalf("Unexpected error deleting pod cluster: %s", err)
}
_, err = store.Get(pc.ID)
if err == nil {
t.Fatalf("Should have gotten an error fetching a deleted pod cluster")
}
if !IsNotExist(err) {
t.Errorf("The error should have been a pocstore.IsNotExist but was '%s'", err)
}
labels, err := store.applicator.GetLabels(labels.PC, pc.ID.String())
if err != nil {
t.Fatalf("Got error when trying to confirm label deletion: %s", err)
}
if len(labels.Labels) != 0 {
t.Errorf("Labels were not deleted along with the pod cluster")
}
}
func updateWithHealth(t *testing.T,
desiredOld, desiredNew int,
oldNodes, newNodes map[types.NodeName]bool,
checks map[types.NodeName]health.Result,
) (update, manifest.Manifest, manifest.Manifest) {
podID := "mypod"
oldManifest := podWithIDAndPort(podID, 9001)
newManifest := podWithIDAndPort(podID, 9002)
podMap := map[kptest.FakePodStoreKey]manifest.Manifest{}
assignManifestsToNodes(types.PodID(podID), oldNodes, podMap, oldManifest, newManifest)
assignManifestsToNodes(types.PodID(podID), newNodes, podMap, newManifest, oldManifest)
kps := kptest.NewFakePodStore(podMap, nil)
rcs := rcstore.NewFake()
applicator := labels.NewFakeApplicator()
oldRC, err := createRC(rcs, applicator, oldManifest, desiredOld, oldNodes)
Assert(t).IsNil(err, "expected no error setting up old RC")
newRC, err := createRC(rcs, applicator, newManifest, desiredNew, newNodes)
Assert(t).IsNil(err, "expected no error setting up new RC")
return update{
kps: kps,
rcs: rcs,
hcheck: checkertest.NewSingleService(podID, checks),
labeler: applicator,
logger: logging.TestLogger(),
Update: fields.Update{
OldRC: oldRC.ID,
NewRC: newRC.ID,
},
}, oldManifest, newManifest
}
func TestList(t *testing.T) {
store := consulStoreWithFakeKV()
podID := types.PodID("pod_id")
az := fields.AvailabilityZone("us-west")
clusterName := fields.ClusterName("cluster_name")
selector := klabels.Everything()
annotations := fields.Annotations(map[string]interface{}{
"foo": "bar",
})
// Create a pod cluster
pc, err := store.Create(podID, az, clusterName, selector, annotations, kptest.NewSession())
if err != nil {
t.Fatalf("Unable to create pod cluster: %s", err)
}
// Create another one
pc2, err := store.Create(podID+"2", az, clusterName, selector, annotations, kptest.NewSession())
if err != nil {
t.Fatalf("Unable to create pod cluster: %s", err)
}
// Now test List() and make sure we get both back
pcs, err := store.List()
if err != nil {
t.Fatalf("Unable to list pod clusters: %s", err)
}
if len(pcs) != 2 {
t.Fatalf("Expected 2 results but there were %d", len(pcs))
}
for _, foundPC := range pcs {
found := false
for _, expectedPC := range []fields.ID{pc.ID, pc2.ID} {
if foundPC.ID == expectedPC {
found = true
}
}
if !found {
t.Errorf("Didn't find one of the pod clusters in the list")
}
}
}
func (h AlwaysHappyHealthChecker) WatchNodeService(
nodeName types.NodeName,
serviceID string,
resultCh chan<- health.Result,
errCh chan<- error,
quitCh <-chan struct{},
) {
happyResult := health.Result{
ID: types.PodID(serviceID),
Status: health.Passing,
}
for {
select {
case <-quitCh:
return
case resultCh <- happyResult:
}
}
}
func (rm *P2RM) deletePod() error {
_, err := rm.Store.DeletePod(kp.INTENT_TREE, rm.NodeName, types.PodID(rm.PodName))
if err != nil {
fmt.Fprintf(os.Stderr, "unable to remove pod: %v", err)
return err
}
err = rm.Labeler.RemoveAllLabels(labels.POD, rm.LabelID)
if err != nil {
fmt.Fprintf(
os.Stderr,
"node is partially deleted. re-run command to finish deleting\n"+
"unable to remove pod labels: %v",
err,
)
return err
}
return nil
}
func TestWatchPodCluster(t *testing.T) {
store := consulStoreWithFakeKV()
pod := fields.ID("pod_id")
podID := types.PodID("pod_id")
az := fields.AvailabilityZone("us-west")
clusterName := fields.ClusterName("cluster_name")
selector := klabels.Everything().
Add(fields.PodIDLabel, klabels.EqualsOperator, []string{pod.String()}).
Add(fields.AvailabilityZoneLabel, klabels.EqualsOperator, []string{az.String()}).
Add(fields.ClusterNameLabel, klabels.EqualsOperator, []string{clusterName.String()})
annotations := fields.Annotations(map[string]interface{}{
"foo": "bar",
})
session := kptest.NewSession()
pc, err := store.Create(podID, az, clusterName, selector, annotations, session)
if err != nil {
t.Fatalf("Unable to create pod cluster: %s", err)
}
quit := make(chan struct{})
watch := store.WatchPodCluster(pc.ID, quit)
var watched *WatchedPodCluster
select {
case watchedPC := <-watch:
watched = &watchedPC
case <-time.After(5 * time.Second):
quit <- struct{}{}
t.Fatal("nothing on the channel")
}
if watched == nil {
t.Fatalf("Expected to get a watched PodCluster, but did not")
}
if watched.PodCluster.ID != pc.ID {
t.Fatalf("Expected watched PodCluster to match %s Pod Cluster ID. Instead was %s", pc.ID, watched.PodCluster.ID)
}
}
func TestPodClusterFromID(t *testing.T) {
testAZ := fields.AvailabilityZone("west-coast")
testCN := fields.ClusterName("test")
testPodID := types.PodID("pod")
selector := labels.Everything().
Add(fields.PodIDLabel, labels.EqualsOperator, []string{testPodID.String()}).
Add(fields.AvailabilityZoneLabel, labels.EqualsOperator, []string{testAZ.String()}).
Add(fields.ClusterNameLabel, labels.EqualsOperator, []string{testCN.String()})
session := kptest.NewSession()
fakePCStore := pcstoretest.NewFake()
pcControllerFromLabels := NewPodCluster(testAZ, testCN, testPodID, fakePCStore, selector, session)
pc, err := pcControllerFromLabels.Create(fields.Annotations{})
if err != nil {
t.Fatal(err)
}
pcControllerFromLabels = nil
pcControllerFromID := NewPodClusterFromID(pc.ID, session, fakePCStore)
retrievedPC, err := pcControllerFromID.Get()
if err != nil {
t.Fatal(err)
}
if pc.ID != retrievedPC.ID {
t.Errorf("Did not get correct PC back from datastore, expected %s, got %s.\n%v", pc.ID, retrievedPC.ID, retrievedPC)
}
errs := pcControllerFromID.Delete()
if len(errs) > 0 {
t.Fatalf("%v", errs)
}
notFoundPC, err := pcControllerFromID.Get()
if err != pcstore.NoPodCluster {
t.Errorf("Expected to get pcstore.NoPodCluster, but got %v", err)
}
if notFoundPC.ID != "" {
t.Errorf("Expected to not find PC but found %v", notFoundPC)
}
}
func (h AlwaysHappyHealthChecker) WatchService(
serviceID string,
resultCh chan<- map[types.NodeName]health.Result,
errCh chan<- error,
quitCh <-chan struct{},
) {
allHappy := make(map[types.NodeName]health.Result)
for _, node := range h.allNodes {
allHappy[node] = health.Result{
ID: types.PodID(serviceID),
Status: health.Passing,
}
}
for {
select {
case <-quitCh:
return
case resultCh <- allHappy:
}
}
}
// Runs Scan() once on the passed Scanner and converts the result to a FinishOutput
func scanRow(scanner Scanner) (FinishOutput, error) {
var id int64
var date time.Time
var podID, podUniqueKey, launchableID, entryPoint string
var exitCode, exitStatus int
err := scanner.Scan(&id, &date, &podID, &podUniqueKey, &launchableID, &entryPoint, &exitCode, &exitStatus)
if err != nil {
return FinishOutput{}, err
}
return FinishOutput{
ID: id,
PodID: types.PodID(podID),
LaunchableID: launch.LaunchableID(launchableID),
EntryPoint: entryPoint,
PodUniqueKey: types.PodUniqueKey(podUniqueKey),
ExitCode: exitCode,
ExitStatus: exitStatus,
ExitTime: date,
}, nil
}
// UpdatePods looks at the pods currently being monitored and
// compares that to what the reality store indicates should be
// running. UpdatePods then shuts down the monitors for dead
// pods and creates PodWatch structs for new pods.
func TestUpdatePods(t *testing.T) {
var current []PodWatch
var reality []kp.ManifestResult
// ids for current: 0, 1, 2, 3
for i := 0; i < 4; i++ {
current = append(current, *newWatch(types.PodID(strconv.Itoa(i))))
}
// ids for reality: 1, 2, test
for i := 1; i < 3; i++ {
reality = append(reality, newManifestResult(current[i].manifest.ID()))
}
reality = append(reality, newManifestResult("test"))
// ids for pods: 1, 2, test
// 0, 3 should have values in their shutdownCh
logger := logging.NewLogger(logrus.Fields{})
pods := updatePods(&MockHealthManager{}, nil, nil, current, reality, "", &logger)
Assert(t).AreEqual(true, <-current[0].shutdownCh, "this PodWatch should have been shutdown")
Assert(t).AreEqual(true, <-current[3].shutdownCh, "this PodWatch should have been shutdown")
Assert(t).AreEqual(current[1].manifest.ID(), pods[0].manifest.ID(), "pod with id:1 should have been returned")
Assert(t).AreEqual(current[2].manifest.ID(), pods[1].manifest.ID(), "pod with id:1 should have been returned")
Assert(t).AreEqual("test", string(pods[2].manifest.ID()), "should have added pod with id:test to list")
}
func main() {
kingpin.Version(version.VERSION)
_, opts, _ := flags.ParseWithConsulOptions()
client := kp.NewConsulClient(opts)
store := kp.NewConsulStore(client)
var intents []kp.ManifestResult
var realities []kp.ManifestResult
var err error
filterNodeName := types.NodeName(*nodeArg)
filterPodID := types.PodID(*podArg)
if filterNodeName != "" {
intents, _, err = store.ListPods(kp.INTENT_TREE, filterNodeName)
} else {
intents, _, err = store.AllPods(kp.INTENT_TREE)
}
if err != nil {
message := "Could not list intent kvpairs: %s"
if kvErr, ok := err.(consulutil.KVError); ok {
log.Fatalf(message, kvErr.UnsafeError)
} else {
log.Fatalf(message, err)
}
}
if filterNodeName != "" {
realities, _, err = store.ListPods(kp.REALITY_TREE, filterNodeName)
} else {
realities, _, err = store.AllPods(kp.REALITY_TREE)
}
if err != nil {
message := "Could not list reality kvpairs: %s"
if kvErr, ok := err.(consulutil.KVError); ok {
log.Fatalf(message, kvErr.UnsafeError)
} else {
log.Fatalf(message, err)
}
}
statusMap := make(map[types.PodID]map[types.NodeName]inspect.NodePodStatus)
for _, kvp := range intents {
if err = inspect.AddKVPToMap(kvp, inspect.INTENT_SOURCE, filterNodeName, filterPodID, statusMap); err != nil {
log.Fatal(err)
}
}
for _, kvp := range realities {
if err = inspect.AddKVPToMap(kvp, inspect.REALITY_SOURCE, filterNodeName, filterPodID, statusMap); err != nil {
log.Fatal(err)
}
}
hchecker := checker.NewConsulHealthChecker(client)
for podID := range statusMap {
resultMap, err := hchecker.Service(podID.String())
if err != nil {
log.Fatalf("Could not retrieve health checks for pod %s: %s", podID, err)
}
for node, result := range resultMap {
if filterNodeName != "" && node != filterNodeName {
continue
}
old := statusMap[podID][node]
old.Health = result.Status
statusMap[podID][node] = old
}
}
// Keep this switch in sync with the enum options for the "format" flag. Rethink this
// design once there are many different formats.
switch *format {
case "tree":
// Native data format is already a "tree"
enc := json.NewEncoder(os.Stdout)
err = enc.Encode(statusMap)
case "list":
// "List" format is a flattened version of "tree"
var output []inspect.NodePodStatus
for podID, nodes := range statusMap {
for node, status := range nodes {
status.PodId = podID
status.NodeName = node
output = append(output, status)
}
}
enc := json.NewEncoder(os.Stdout)
err = enc.Encode(output)
default:
err = fmt.Errorf("unrecognized format: %s", *format)
}
if err != nil {
log.Fatal(err)
}
}
func main() {
cmd, consulOpts := flags.ParseWithConsulOptions()
client := kp.NewConsulClient(consulOpts)
logger := logging.NewLogger(logrus.Fields{})
dsstore := dsstore.NewConsul(client, 3, &logger)
applicator := labels.NewConsulApplicator(client, 3)
switch cmd {
case CmdCreate:
minHealth, err := strconv.Atoi(*createMinHealth)
if err != nil {
log.Fatalf("Invalid value for minimum health, expected integer: %v", err)
}
name := ds_fields.ClusterName(*createName)
manifest, err := manifest.FromPath(*createManifest)
if err != nil {
log.Fatalf("%s", err)
}
podID := manifest.ID()
if *createTimeout <= time.Duration(0) {
log.Fatalf("Timeout must be a positive non-zero value, got '%v'", *createTimeout)
}
selectorString := *createSelector
if *createEverywhere {
selectorString = klabels.Everything().String()
} else if selectorString == "" {
selectorString = klabels.Nothing().String()
log.Fatal("Explicit everything selector not allowed, please use the --everwhere flag")
}
selector, err := parseNodeSelectorWithPrompt(klabels.Nothing(), selectorString, applicator)
if err != nil {
log.Fatalf("Error occurred: %v", err)
}
if err = confirmMinheathForSelector(minHealth, selector, applicator); err != nil {
log.Fatalf("Error occurred: %v", err)
}
ds, err := dsstore.Create(manifest, minHealth, name, selector, podID, *createTimeout)
if err != nil {
log.Fatalf("err: %v", err)
}
fmt.Printf("%v has been created in consul", ds.ID)
fmt.Println()
case CmdGet:
id := ds_fields.ID(*getID)
ds, _, err := dsstore.Get(id)
if err != nil {
log.Fatalf("err: %v", err)
}
bytes, err := json.Marshal(ds)
if err != nil {
logger.WithError(err).Fatalln("Unable to marshal daemon set as JSON")
}
fmt.Printf("%s", bytes)
case CmdList:
dsList, err := dsstore.List()
if err != nil {
log.Fatalf("err: %v", err)
}
podID := types.PodID(*listPod)
for _, ds := range dsList {
if *listPod == "" || podID == ds.PodID {
fmt.Printf("%s/%s:%s\n", ds.PodID, ds.Name, ds.ID)
}
}
case CmdEnable:
id := ds_fields.ID(*enableID)
mutator := func(ds ds_fields.DaemonSet) (ds_fields.DaemonSet, error) {
if !ds.Disabled {
return ds, util.Errorf("Daemon set has already been enabled")
}
ds.Disabled = false
return ds, nil
}
_, err := dsstore.MutateDS(id, mutator)
if err != nil {
log.Fatalf("err: %v", err)
}
fmt.Printf("The daemon set '%s' has been successfully enabled in consul", id.String())
fmt.Println()
case CmdDisable:
id := ds_fields.ID(*disableID)
mutator := func(ds ds_fields.DaemonSet) (ds_fields.DaemonSet, error) {
if ds.Disabled {
return ds, util.Errorf("Daemon set has already been disabled")
}
ds.Disabled = true
return ds, nil
}
_, err := dsstore.MutateDS(id, mutator)
if err != nil {
log.Fatalf("err: %v", err)
}
fmt.Printf("The daemon set '%s' has been successfully disabled in consul", id.String())
fmt.Println()
case CmdDelete:
id := ds_fields.ID(*deleteID)
err := dsstore.Delete(id)
if err != nil {
log.Fatalf("err: %v", err)
}
fmt.Printf("The daemon set '%s' has been successfully deleted from consul", id.String())
fmt.Println()
case CmdUpdate:
id := ds_fields.ID(*updateID)
mutator := func(ds ds_fields.DaemonSet) (ds_fields.DaemonSet, error) {
changed := false
if *updateMinHealth != "" {
minHealth, err := strconv.Atoi(*updateMinHealth)
if err != nil {
log.Fatalf("Invalid value for minimum health, expected integer")
}
if ds.MinHealth != minHealth {
changed = true
ds.MinHealth = minHealth
}
}
if *updateName != "" {
name := ds_fields.ClusterName(*updateName)
if ds.Name != name {
changed = true
ds.Name = name
}
}
if *updateTimeout != TimeoutNotSpecified {
if *updateTimeout <= time.Duration(0) {
return ds, util.Errorf("Timeout must be a positive non-zero value, got '%v'", *createTimeout)
}
if ds.Timeout != *updateTimeout {
changed = true
ds.Timeout = *updateTimeout
}
}
if *updateManifest != "" {
manifest, err := manifest.FromPath(*updateManifest)
if err != nil {
return ds, util.Errorf("%s", err)
}
if manifest.ID() != ds.PodID {
return ds, util.Errorf("Manifest ID of %s does not match daemon set's pod ID (%s)", manifest.ID(), ds.PodID)
}
dsSHA, err := ds.Manifest.SHA()
if err != nil {
return ds, util.Errorf("Unable to get SHA from consul daemon set manifest: %v", err)
}
newSHA, err := manifest.SHA()
if err != nil {
return ds, util.Errorf("Unable to get SHA from new manifest: %v", err)
}
if dsSHA != newSHA {
changed = true
ds.Manifest = manifest
}
}
if updateSelectorGiven {
selectorString := *updateSelector
if *updateEverywhere {
selectorString = klabels.Everything().String()
} else if selectorString == "" {
return ds, util.Errorf("Explicit everything selector not allowed, please use the --everwhere flag")
}
selector, err := parseNodeSelectorWithPrompt(ds.NodeSelector, selectorString, applicator)
if err != nil {
return ds, util.Errorf("Error occurred: %v", err)
}
if ds.NodeSelector.String() != selector.String() {
changed = true
ds.NodeSelector = selector
}
}
if !changed {
return ds, util.Errorf("No changes were made")
}
if updateSelectorGiven || *updateMinHealth != "" {
if err := confirmMinheathForSelector(ds.MinHealth, ds.NodeSelector, applicator); err != nil {
return ds, util.Errorf("Error occurred: %v", err)
}
}
return ds, nil
}
_, err := dsstore.MutateDS(id, mutator)
if err != nil {
log.Fatalf("err: %v", err)
}
fmt.Printf("The daemon set '%s' has been successfully updated in consul", id.String())
fmt.Println()
case CmdTestSelector:
selectorString := *testSelectorString
if *testSelectorEverywhere {
selectorString = klabels.Everything().String()
} else if selectorString == "" {
fmt.Println("Explicit everything selector not allowed, please use the --everwhere flag")
}
selector, err := parseNodeSelector(selectorString)
if err != nil {
log.Fatalf("Error occurred: %v", err)
}
matches, err := applicator.GetMatches(selector, labels.NODE, false)
if err != nil {
log.Fatalf("Error getting matching labels: %v", err)
}
fmt.Println(matches)
default:
log.Fatalf("Unrecognized command %v", cmd)
}
}
func TestConcreteSyncerWithPrevious(t *testing.T) {
store := consulStoreWithFakeKV()
store.logger.Logger.Level = logrus.DebugLevel
store.applicator.SetLabel(labels.POD, "1234-123-123-1234", "color", "red")
store.applicator.SetLabel(labels.POD, "abcd-abc-abc-abcd", "color", "blue")
syncer := &fakeSyncer{
[]fields.ID{},
make(chan fakeSync),
make(chan fakeSync),
false,
}
// Previous == current, simulates a concrete syncer starting up
change := podClusterChange{
previous: &fields.PodCluster{
ID: fields.ID("abc123"),
PodID: types.PodID("vvv"),
AvailabilityZone: fields.AvailabilityZone("west"),
Name: "production",
PodSelector: klabels.Everything().Add("color", klabels.EqualsOperator, []string{"red"}),
},
current: &fields.PodCluster{
ID: fields.ID("abc123"),
PodID: types.PodID("vvv"),
AvailabilityZone: fields.AvailabilityZone("west"),
Name: "production",
PodSelector: klabels.Everything().Add("color", klabels.EqualsOperator, []string{"red"}),
},
}
changes := make(chan podClusterChange)
go store.handlePCUpdates(syncer, changes, metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015)))
select {
case changes <- change:
case <-time.After(5 * time.Second):
t.Fatal("Test timed out trying to write change to handlePCChange")
}
select {
case <-time.After(5 * time.Second):
t.Fatal("Test timed out trying to read from the syncer")
case sync := <-syncer.synced:
if sync.syncedCluster == nil {
t.Fatal("unexpectedly didn't get a cluster on the sync channel")
}
if sync.syncedCluster.ID != change.current.ID {
t.Fatalf("got unexpected synced cluster %v", sync.syncedCluster.ID)
}
if len(sync.syncedPods) != 1 {
t.Fatalf("got unexpected number of synced pods with cluster: %v", len(sync.syncedPods))
}
if sync.syncedPods[0].ID != "1234-123-123-1234" {
t.Fatalf("got unexpected pod ID from labeled pods sync: %v", sync.syncedPods[0].ID)
}
}
// now we send a new update that changes the pod cluster's target pod from the red one to the blue one.
// (from 1234-123-123-1234 to abcd-abc-abc-abcd )
change = podClusterChange{
previous: change.current,
current: &fields.PodCluster{
ID: fields.ID("abc123"),
PodID: types.PodID("vvv"),
AvailabilityZone: fields.AvailabilityZone("west"),
Name: "production",
PodSelector: klabels.Everything().Add("color", klabels.EqualsOperator, []string{"blue"}),
},
}
select {
case changes <- change:
case <-time.After(5 * time.Second):
t.Fatal("Test timed out trying to write change to handlePCChange")
}
select {
case <-time.After(5 * time.Second):
t.Fatal("Test timed out trying to read from the syncer")
case sync := <-syncer.synced:
if sync.syncedCluster == nil {
t.Fatal("unexpectedly didn't get a cluster on the sync channel")
}
if sync.syncedCluster.ID != change.current.ID {
t.Fatalf("got unexpected synced cluster %v", sync.syncedCluster.ID)
}
if len(sync.syncedPods) != 1 {
t.Fatalf("got unexpected number of synced pods with cluster: %v", len(sync.syncedPods))
}
if sync.syncedPods[0].ID != "abcd-abc-abc-abcd" {
t.Fatalf("got unexpected pod ID from labeled pods sync: %v", sync.syncedPods[0].ID)
}
}
// appear to have deleted the cluster
change.previous = change.current
change.current = nil
select {
case changes <- change:
case <-time.After(5 * time.Second):
t.Fatal("Test timed out trying to write deletion change to handlePCChange")
}
select {
case <-time.After(5 * time.Second):
t.Fatal("Test timed out trying to read from the syncer")
case sync := <-syncer.deleted:
if sync.syncedCluster == nil {
t.Fatal("unexpectedly didn't get a cluster on the sync channel")
}
if sync.syncedCluster.ID != change.previous.ID {
t.Fatalf("got unexpected synced cluster %v", sync.syncedCluster.ID)
}
}
close(changes)
}