func NewReplicator(
manifest manifest.Manifest,
logger logging.Logger,
nodes []types.NodeName,
active int,
store kp.Store,
labeler Labeler,
health checker.ConsulHealthChecker,
threshold health.HealthState,
lockMessage string,
timeout time.Duration,
) (Replicator, error) {
if active < 1 {
return replicator{}, util.Errorf("Active must be >= 1, was %d", active)
}
if active > 50 {
logger.Infof("Number of concurrent updates (%v) is greater than 50, reducing to 50", active)
active = 50
}
return replicator{
manifest: manifest,
logger: logger,
nodes: nodes,
active: active,
store: store,
labeler: labeler,
health: health,
threshold: threshold,
lockMessage: lockMessage,
timeout: timeout,
}, nil
}
func schedule(client client.Client, logger logging.Logger) {
m, err := manifest.FromPath(*manifestFile)
if err != nil {
logger.Fatalf("Could not read manifest: %s", err)
}
podUniqueKey, err := client.Schedule(m, types.NodeName(*node))
if err != nil {
logger.Fatalf("Could not schedule: %s", err)
}
output := struct {
PodID types.PodID `json:"pod_id"`
PodUniqueKey types.PodUniqueKey `json:"pod_unique_key"`
}{
PodID: m.ID(),
PodUniqueKey: podUniqueKey,
}
outBytes, err := json.Marshal(output)
if err != nil {
logger.Infof("Scheduled pod with key: %s", podUniqueKey)
return
}
fmt.Println(string(outBytes))
}
func createHelloUUIDPod(dir string, port int, logger logging.Logger) (types.PodUniqueKey, error) {
signedManifestPath, err := writeHelloManifest(dir, fmt.Sprintf("hello-uuid-%d.yaml", port), port)
if err != nil {
return "", err
}
logger.Infoln("Scheduling uuid pod")
cmd := exec.Command("p2-schedule", "--uuid-pod", signedManifestPath)
stdout := bytes.Buffer{}
stderr := bytes.Buffer{}
cmd.Stdout, cmd.Stderr = &stdout, &stderr
err = cmd.Run()
if err != nil {
fmt.Println(stderr.String())
return "", err
}
var out schedule.Output
err = json.Unmarshal(stdout.Bytes(), &out)
if err != nil {
return "", util.Errorf("Scheduled uuid pod but couldn't parse uuid from p2-schedule output: %s", err)
}
logger.Infof("Scheduled uuid pod %s", out.PodUniqueKey)
return out.PodUniqueKey, nil
}
func verifyLegacyPod(errCh chan error, tempDir string, config *preparer.PreparerConfig, services []string, logger logging.Logger) {
defer close(errCh)
// Schedule a "legacy" hello pod using a replication controller
rcID, err := createHelloReplicationController(tempDir)
if err != nil {
errCh <- fmt.Errorf("Could not create hello pod / rc: %s", err)
return
}
logger.Infof("Created RC #%s for hello\n", rcID)
err = waitForPodLabeledWithRC(klabels.Everything().Add(rc.RCIDLabel, klabels.EqualsOperator, []string{rcID.String()}), rcID)
if err != nil {
errCh <- fmt.Errorf("Failed waiting for pods labeled with the given RC: %v", err)
return
}
err = verifyHelloRunning("", logger)
if err != nil {
errCh <- fmt.Errorf("Couldn't get hello running: %s", err)
return
}
err = verifyHealthChecks(config, services)
if err != nil {
errCh <- fmt.Errorf("Could not get health check info from consul: %s", err)
return
}
}
func verifyProcessExit(errCh chan error, tempDir string, logger logging.Logger) {
defer close(errCh)
// Schedule a uuid pod
podUniqueKey, err := createHelloUUIDPod(tempDir, 43772, logger)
if err != nil {
errCh <- fmt.Errorf("Could not schedule UUID hello pod: %s", err)
return
}
logger = logger.SubLogger(logrus.Fields{
"pod_unique_key": podUniqueKey,
})
logger.Infoln("Scheduled hello instance on port 43772")
err = verifyHelloRunning(podUniqueKey, logger)
if err != nil {
errCh <- fmt.Errorf("Couldn't get hello running as a uuid pod: %s", err)
return
}
logger.Infoln("Hello instance launched")
time.Sleep(3 * time.Second)
logger.Infoln("Waiting for hello instance to listen on 43772")
// now wait for the hello server to start running
timeout := time.After(30 * time.Second)
for {
resp, err := http.Get("http://localhost:43772/")
if err == nil {
resp.Body.Close()
break
}
select {
case <-timeout:
errCh <- fmt.Errorf("Hello didn't come up listening on 43772: %s", err)
return
default:
}
time.Sleep(1 * time.Second)
}
exitCode := rand.Intn(100) + 1
logger.Infof("Causing hello on 43772 to exit with status %d", exitCode)
// Make an http request to hello to make it exit with exitCode. We expect the http request to fail due
// to the server exiting, so don't check for http errors.
_, err = http.Get(fmt.Sprintf("http://localhost:43772/exit/%d", exitCode))
if err == nil {
// This is bad, it means the hello server didn't die and kill our request
// in the middle
errCh <- util.Errorf("Couldn't kill hello server with http request")
return
}
urlError, ok := err.(*url.Error)
if ok && urlError.Err == io.EOF {
// This is good, it means the server died
} else {
errCh <- fmt.Errorf("Couldn't tell hello to die over http: %s", err)
return
}
logger.Infoln("Checking for exit code in SQL database")
finishService, err := podprocess.NewSQLiteFinishService(sqliteFinishDatabasePath, logging.DefaultLogger)
if err != nil {
errCh <- err
return
}
var finishResult podprocess.FinishOutput
timeout = time.After(30 * time.Second)
for {
finishResult, err = finishService.LastFinishForPodUniqueKey(podUniqueKey)
if err == nil {
break
}
select {
case <-timeout:
// Try to manually run the finish script in order to make debugging the test failure easier
output, err := exec.Command("sudo", fmt.Sprintf("/var/service/hello-%s__hello__launch/finish", podUniqueKey), "1", "2").CombinedOutput()
if err != nil {
logger.WithError(err).Infoln("DEBUG: Debug attempt to run finish script failed")
}
logger.Infof("DEBUG: Output of direct execution of finish script: %s", string(output))
errCh <- fmt.Errorf("Did not find a finish row by the deadline: %s", err)
return
default:
}
}
if finishResult.PodUniqueKey != podUniqueKey {
errCh <- fmt.Errorf("Expected finish result for '%s' but it was for '%s'", podUniqueKey, finishResult.PodUniqueKey)
return
}
if finishResult.ExitCode != exitCode {
errCh <- fmt.Errorf("Exit code for '%s' in the sqlite database was expected to be %d but was %d", podUniqueKey, exitCode, finishResult.ExitCode)
return
}
logger.Infoln("Checking for exit code in consul")
timeout = time.After(30 * time.Second)
podStatusStore := podstatus.NewConsul(statusstore.NewConsul(kp.NewConsulClient(kp.Options{})), kp.PreparerPodStatusNamespace)
for {
podStatus, _, err := podStatusStore.Get(podUniqueKey)
if err != nil {
errCh <- err
return
}
found := false
for _, processStatus := range podStatus.ProcessStatuses {
if processStatus.LaunchableID == "hello" && processStatus.EntryPoint == "launch" {
found = true
if processStatus.LastExit == nil {
errCh <- fmt.Errorf("Found no last exit in consul pod status for %s", podUniqueKey)
return
}
if processStatus.LastExit.ExitCode != exitCode {
errCh <- fmt.Errorf("Exit code for '%s' in consul was expected to be %d but was %d", podUniqueKey, exitCode, finishResult.ExitCode)
return
}
}
}
if found {
logger.Infoln("Successful!")
break
}
select {
case <-timeout:
errCh <- fmt.Errorf("There was no pod process for hello/launch for %s in consul", podUniqueKey)
return
default:
}
}
}
