func TestSetAndGetStatus(t *testing.T) { store := newFixture() serviceStatus := ServiceStatus{ Name: "echo_service", State: Running, LastExit: nil, } testStatus := PodStatus{ ServiceStatus: []ServiceStatus{ serviceStatus, }, } podKey := types.NewPodUUID() err := store.Set(podKey, testStatus) if err != nil { t.Fatalf("Unexpected error setting status: %s", err) } status, _, err := store.Get(podKey) if err != nil { t.Fatalf("Unexpected error getting status: %s", err) } if len(status.ServiceStatus) != 1 { t.Fatalf("Expected one service status entry, but there were %d", len(status.ServiceStatus)) } if status.ServiceStatus[0] != serviceStatus { t.Errorf("Status entry expected to be '%+v', was %+v", serviceStatus, status.ServiceStatus[0]) } }
func TestSetAndGetStatus(t *testing.T) { store := newFixture() processStatus := ProcessStatus{ EntryPoint: "echo_service", LaunchableID: "some_launchable", LastExit: nil, } testStatus := PodStatus{ ProcessStatuses: []ProcessStatus{ processStatus, }, } podKey := types.NewPodUUID() err := store.Set(podKey, testStatus) if err != nil { t.Fatalf("Unexpected error setting status: %s", err) } status, _, err := store.Get(podKey) if err != nil { t.Fatalf("Unexpected error getting status: %s", err) } if len(status.ProcessStatuses) != 1 { t.Fatalf("Expected one service status entry, but there were %d", len(status.ProcessStatuses)) } if status.ProcessStatuses[0] != processStatus { t.Errorf("Status entry expected to be '%+v', was %+v", processStatus, status.ProcessStatuses[0]) } }
func TestWriteRealityIndex(t *testing.T) { node := types.NodeName("some_node") key := types.NewPodUUID() realityIndexPath := fmt.Sprintf("reality/%s/%s", node, key.ID) store, fakeKV := storeWithFakeKV(t, make(map[string]Pod), make(map[string]PodIndex)) // confirm that the reality index doesn't exist pair, _, err := fakeKV.Get(realityIndexPath, nil) if err != nil { t.Fatalf("Initial conditions were not met: error fetching %s: %s", realityIndexPath, err) } if pair != nil { t.Fatalf("Initial conditions were not met: expected key %s to not exist", realityIndexPath) } err = store.WriteRealityIndex(key, node) if err != nil { t.Fatal(err) } pair, _, err = fakeKV.Get(realityIndexPath, nil) if err != nil { t.Fatalf("Unable to fetch the deleted key (%s): %s", realityIndexPath, err) } if pair == nil { t.Fatalf("%s should have been written but it wasn't", realityIndexPath) } }
// UpdatePods looks at the pods currently being monitored and // compares that to what the reality store indicates should be // running. UpdatePods then shuts down the monitors for dead // pods and creates PodWatch structs for new pods. func TestUpdatePods(t *testing.T) { var current []PodWatch var reality []kp.ManifestResult // ids for current: 0, 1, 2, 3 for i := 0; i < 4; i++ { current = append(current, *newWatch(types.PodID(strconv.Itoa(i)))) } // ids for reality: 1, 2, test for i := 1; i < 3; i++ { // Health checking is not supported for uuid pods, so ensure that even // if /reality contains a uuid pod we don't actually watch its health uuidKeyResult := newManifestResult("some_uuid_pod") uuidKeyResult.PodUniqueKey = types.NewPodUUID() reality = append(reality, uuidKeyResult) reality = append(reality, newManifestResult(current[i].manifest.ID())) } reality = append(reality, newManifestResult("test")) // ids for pods: 1, 2, test // 0, 3 should have values in their shutdownCh logger := logging.NewLogger(logrus.Fields{}) pods := updatePods(&MockHealthManager{}, nil, nil, current, reality, "", &logger) Assert(t).AreEqual(true, <-current[0].shutdownCh, "this PodWatch should have been shutdown") Assert(t).AreEqual(true, <-current[3].shutdownCh, "this PodWatch should have been shutdown") Assert(t).AreEqual(current[1].manifest.ID(), pods[0].manifest.ID(), "pod with id:1 should have been returned") Assert(t).AreEqual(current[2].manifest.ID(), pods[1].manifest.ID(), "pod with id:1 should have been returned") Assert(t).AreEqual("test", string(pods[2].manifest.ID()), "should have added pod with id:test to list") }
func TestRun(t *testing.T) { tempDir, err := ioutil.TempDir("", "process_reporter") if err != nil { t.Fatalf("Could not create temp dir: %s", err) } defer os.RemoveAll(tempDir) dbPath, quitCh, podStatusStore := startReporter(t, tempDir) defer close(quitCh) finishOutput1 := FinishOutput{ PodID: "some_pod", LaunchableID: "some_launchable", EntryPoint: "launch", PodUniqueKey: types.NewPodUUID(), ExitCode: 1, ExitStatus: 120, } finishService, err := NewSQLiteFinishService(dbPath, logging.DefaultLogger) if err != nil { t.Fatalf("Could not initialize finish service: %s", err) } defer finishService.Close() err = finishService.Insert(finishOutput1) if err != nil { t.Fatalf("Could not insert first finish value into the database: %s", err) } assertStatusUpdated(t, finishOutput1, podStatusStore) finishOutput2 := FinishOutput{ PodID: "some_pod", LaunchableID: "some_launchable", EntryPoint: "nginx_worker", PodUniqueKey: types.NewPodUUID(), ExitCode: 3, ExitStatus: 67, } err = finishService.Insert(finishOutput2) if err != nil { t.Fatalf("Could not insert second finish value into the database: %s", err) } assertStatusUpdated(t, finishOutput2, podStatusStore) }
func TestPodUniqueKeyFromConsulPath(t *testing.T) { type expectation struct { path string err bool uuid types.PodUniqueKey } uuid := types.NewPodUUID() expectations := []expectation{ { path: "intent/example.com/mysql", err: false, uuid: "", }, { path: "reality/example.com/mysql", err: false, uuid: "", }, { path: "labels/example.com/mysql", err: true, }, { path: "intent/example/com/mysql", err: true, }, { path: "hooks/all_hooks", err: false, uuid: "", }, { path: fmt.Sprintf("intent/example.com/%s", uuid), uuid: uuid, err: false, }, } for _, expectation := range expectations { podUniqueKey, err := PodUniqueKeyFromConsulPath(expectation.path) if expectation.err { if err == nil { t.Errorf("Expected an error for key '%s'", expectation.path) } continue } if err != nil { t.Errorf("Unexpected error for key '%s': %s", expectation.path, err) continue } if podUniqueKey != expectation.uuid { t.Errorf("Expected podUniqueKey to be %s, was %s", expectation.uuid, podUniqueKey) } } }
func TestPruneRowsBefore(t *testing.T) { finishService, dbPath, closeFunc := initFinishService(t) defer closeFunc() defer finishService.Close() // Insert 3 rows for i := 0; i < 3; i++ { // now check that we can insert a finish to confirm finishes table was created err := finishService.Insert(FinishOutput{ PodID: "some_pod", PodUniqueKey: types.NewPodUUID(), LaunchableID: "some_launchable", EntryPoint: "launch", ExitCode: 4, ExitStatus: 127, }) if err != nil { t.Errorf("Could not insert a finish row, the finishes table might not have been created: %s", err) } } // Open up the DB directly so we can mess with the dates db, err := sql.Open("sqlite3", dbPath) if err != nil { t.Fatalf("Could not open database to confirm migrations worked: %s", err) } defer db.Close() // Set the dates of the first two rows to a week in the past _, err = db.Exec(` update finishes set date = ? where id < 3 `, time.Now().Add(-7*24*time.Hour)) if err != nil { t.Fatalf("Could not set time back a week for two rows: %s", err) } // Now prune any rows older than an hour, we should see the first two deleted err = finishService.PruneRowsBefore(time.Now().Add(-time.Hour)) if err != nil { t.Fatalf("Unexpected error pruning rows: %s", err) } finishes, err := finishService.GetLatestFinishes(0) if err != nil { t.Fatalf("Could not fetch all finishes from the table to verify pruning: %s", err) } if len(finishes) != 1 { t.Fatalf("Expected only 1 row after pruning, but there were %d", len(finishes)) } }
func TestReadPodFromIndex(t *testing.T) { node := types.NodeName("some_node") key := types.NewPodUUID() podPath := fmt.Sprintf("pods/%s", key.ID) indexPath := fmt.Sprintf("intent/%s/%s", node, key.ID) pod := Pod{ Manifest: testManifest(), Node: node, } index := PodIndex{ PodKey: key, } // Initialize the store with entries at the pod path and index path pods := map[string]Pod{ podPath: pod, } // This test doesn't actually care if there's an index, but let's keep it hygienic indices := map[string]PodIndex{ indexPath: index, } store, _ := storeWithFakeKV(t, pods, indices) outPod, err := store.ReadPod(key) if err != nil { t.Fatalf("Unexpected error reading pod: %s", err) } if pod.Node != outPod.Node { t.Errorf("Pod node, didn't match expected, wanted %+v was %+v", pod.Node, outPod.Node) } expectedManifestSha, err := testManifest().SHA() if err != nil { t.Fatal(err) } podManifestSha, err := outPod.Manifest.SHA() if err != nil { t.Fatal(err) } if expectedManifestSha != podManifestSha { t.Error("Pod returned with the wrong manifest") } }
func TestMutateStatusExistingKey(t *testing.T) { store := newFixture() key := types.NewPodUUID() processStatus := ProcessStatus{ EntryPoint: "echo_service", LaunchableID: "some_launchable", LastExit: nil, } err := store.Set(key, PodStatus{ ProcessStatuses: []ProcessStatus{ processStatus, }, }) if err != nil { t.Fatalf("Unable to set up test with an existing key: %s", err) } err = store.MutateStatus(key, func(p PodStatus) (PodStatus, error) { p.PodStatus = PodLaunched return p, nil }) if err != nil { t.Fatal(err) } // Now try to get it and confirm the status was set status, _, err := store.Get(key) if err != nil { t.Fatal(err) } if status.PodStatus != PodLaunched { t.Errorf("Expected pod status to be set to '%s' but was '%s'", PodLaunched, status.PodStatus) } if len(status.ProcessStatuses) != 1 { t.Error("ProcessStatus field didn't go untouched when mutating PodStatus") } }
func TestMutateStatusNewKey(t *testing.T) { store := newFixture() key := types.NewPodUUID() err := store.MutateStatus(key, func(p PodStatus) (PodStatus, error) { p.PodStatus = PodLaunched return p, nil }) if err != nil { t.Fatal(err) } // Now try to get it and confirm the status was set status, _, err := store.Get(key) if err != nil { t.Fatal(err) } if status.PodStatus != PodLaunched { t.Errorf("Expected pod status to be set to '%s' but was '%s'", PodLaunched, status.PodStatus) } }
func TestMigrate(t *testing.T) { finishService, dbPath, closeFunc := initFinishService(t) defer closeFunc() defer finishService.Close() // now check that we can insert a finish to confirm finishes table was created err := finishService.Insert(FinishOutput{ PodID: "some_pod", PodUniqueKey: types.NewPodUUID(), LaunchableID: "some_launchable", EntryPoint: "launch", ExitCode: 4, ExitStatus: 127, }) if err != nil { t.Errorf("Could not insert a finish row, the finishes table might not have been created: %s", err) } db, err := sql.Open("sqlite3", dbPath) if err != nil { t.Fatalf("Could not open database to confirm migrations worked: %s", err) } defer db.Close() // now check schema version var schemaVersion int64 err = db.QueryRow(getSchemaVersionQuery).Scan(&schemaVersion) switch { case err == sql.ErrNoRows: t.Fatal("Schema version table was not written") case err != nil: t.Fatalf("Could not check schema version: %s", err) } if schemaVersion != int64(len(sqliteMigrations)) { t.Errorf("Expected schema_version to be %d but was %d", len(sqliteMigrations), schemaVersion) } }
func TestUnschedule(t *testing.T) { node := types.NodeName("some_node") key := types.NewPodUUID() podPath := fmt.Sprintf("pods/%s", key.ID) indexPath := fmt.Sprintf("intent/%s/%s", node, key.ID) // Initialize the store with entries at the pod path and index path pods := map[string]Pod{ podPath: { Manifest: testManifest(), Node: node, }, } indices := map[string]PodIndex{ indexPath: { PodKey: key, }, } store, kv := storeWithFakeKV(t, pods, indices) // Now delete the pod entry err := store.Unschedule(key) if err != nil { t.Fatalf("Unexpected error deleting pod: %s", err) } if kv.Entries[podPath] != nil { t.Fatalf("Key '%s' was deleted as expected", podPath) } if kv.Entries[indexPath] != nil { t.Fatalf("Index '%s' was deleted as expected", indexPath) } }
func TestWatchPodStatus(t *testing.T) { respCh := make(chan *podstore_protos.PodStatusResponse) stream := &WatchPodStatusStream{ FakeServerStream: testutil.NewFakeServerStream(context.Background()), ResponseCh: respCh, } podStatusStore, server := setupServerWithFakePodStatusStore() podUniqueKey := types.NewPodUUID() req := &podstore_protos.WatchPodStatusRequest{ StatusNamespace: kp.PreparerPodStatusNamespace.String(), PodUniqueKey: podUniqueKey.String(), // Set it 1 above last index so we wait for the key to exist. (The test status // store starts at 1234 for some reason) WaitIndex: 1235, } watchErrCh := make(chan error) defer close(watchErrCh) go func() { err := server.WatchPodStatus(req, stream) if err != nil { watchErrCh <- err } }() expectedTime := time.Now() expectedManifest := `id: "test_app"` expectedPodState := podstatus.PodLaunched expectedLaunchableID := launch.LaunchableID("nginx") expectedEntryPoint := "launch" expectedExitCode := 3 expectedExitStatus := 4 setStatusErrCh := make(chan error) defer close(setStatusErrCh) go func() { err := podStatusStore.Set(podUniqueKey, podstatus.PodStatus{ Manifest: expectedManifest, PodStatus: expectedPodState, ProcessStatuses: []podstatus.ProcessStatus{ { LaunchableID: expectedLaunchableID, EntryPoint: expectedEntryPoint, LastExit: &podstatus.ExitStatus{ ExitTime: expectedTime, ExitCode: expectedExitCode, ExitStatus: expectedExitStatus, }, }, }, }) if err != nil { setStatusErrCh <- err } }() select { case <-time.After(5 * time.Second): t.Fatal("Didn't receive value after 5 seconds") case err := <-setStatusErrCh: t.Fatalf("Error setting status to trigger watch: %s", err) case err := <-watchErrCh: t.Fatalf("Unexpected error watching for status: %s", err) case resp := <-respCh: if resp.Manifest != expectedManifest { t.Errorf("Manifest didn't match expected, wanted %q got %q", expectedManifest, resp.Manifest) } if resp.PodState != expectedPodState.String() { t.Errorf("PodState didn't match expcted, wanted %q got %q", expectedPodState, resp.PodState) } if len(resp.ProcessStatuses) != 1 { t.Fatalf("Expected 1 process status in pod status but got %d", len(resp.ProcessStatuses)) } processStatus := resp.ProcessStatuses[0] if processStatus.LaunchableId != expectedLaunchableID.String() { t.Errorf("Expected process status for launchable %q but found %q", expectedLaunchableID, processStatus.LaunchableId) } if processStatus.EntryPoint != expectedEntryPoint { t.Errorf("Expected process status for entry point %q but found %q", expectedEntryPoint, processStatus.EntryPoint) } if processStatus.LastExit == nil { t.Fatal("Expected exit information for process") } lastExit := processStatus.LastExit if lastExit.ExitTime != expectedTime.Unix() { t.Error("Exit time for process in status didn't match expected") } if lastExit.ExitCode != int64(expectedExitCode) { t.Errorf("Expected exit code %d but got %d", expectedExitCode, lastExit.ExitCode) } if lastExit.ExitStatus != int64(expectedExitStatus) { t.Errorf("Expected exit status %d but got %d", expectedExitStatus, lastExit.ExitStatus) } } }
func (c *consulStore) Schedule(manifest manifest.Manifest, node types.NodeName) (key types.PodUniqueKey, err error) { manifestBytes, err := manifest.Marshal() if err != nil { return "", err } podKey := types.NewPodUUID() podPath := computePodPath(podKey) intentIndexPath := computeIntentIndexPath(podKey, node) // Write the Pod to /pods/<key> pod := RawPod{ Manifest: string(manifestBytes), Node: node, } podBytes, err := json.Marshal(pod) if err != nil { return "", err } pair := &api.KVPair{ Key: podPath, Value: podBytes, } _, err = c.consulKV.Put(pair, nil) if err != nil { return "", consulutil.NewKVError("put", podPath, err) } // Now, write the secondary index to /intent/<node>/<key> index := PodIndex{ PodKey: podKey, } // NOTE: errors might happen after this point which means we've written // a pod to /pods and we haven't written the corresponding index. In // those cases, we do a single attempt to delete the main pod key. In // the event of a Consul outage it's likely that the cleanup will fail, // so there will be a pod with no secondary index. For that purpose, a // sweeper process is planned to remove pods for which there is no index. // TODO: use a transaction when we can rely upon consul 0.7 defer func() { if err != nil { _, _ = c.consulKV.Delete(podPath, nil) } }() indexBytes, err := json.Marshal(index) if err != nil { return "", util.Errorf("Could not marshal index as json: %s", err) } indexPair := &api.KVPair{ Key: intentIndexPath, Value: indexBytes, } _, err = c.consulKV.Put(indexPair, nil) if err != nil { return "", consulutil.NewKVError("put", intentIndexPath, err) } return podKey, nil }