func TestNew(t *testing.T) {
reporter, err := New(ReporterConfig{}, logging.DefaultLogger, podstatus.NewConsul(statusstoretest.NewFake(), kp.PreparerPodStatusNamespace))
if reporter != nil || err == nil {
t.Errorf("Should have gotten a nil reporter and an error with empty config")
}
reporter, err = New(ReporterConfig{
SQLiteDatabasePath: "bar",
EnvironmentExtractorPath: "/some/nonexistent/path",
}, logging.DefaultLogger, podstatus.NewConsul(statusstoretest.NewFake(), kp.PreparerPodStatusNamespace))
if reporter != nil || err == nil {
t.Errorf("Should have gotten a nil reporter when EnvironmentExtractorPath doesn't exist")
}
tempDir, err := ioutil.TempDir("", "podprocess_reporter_tests")
if err != nil {
t.Fatalf("Could not set up temp directory for tests: %s", err)
}
defer os.RemoveAll(tempDir)
// os.Create() creates a file with mode 0666, which is not executable
nonExecutableExtractor, err := os.Create(filepath.Join(tempDir, "non_executable_extractor"))
if err != nil {
t.Fatalf("Could not create non-executable extractor file")
}
reporter, err = New(ReporterConfig{
SQLiteDatabasePath: "foo",
EnvironmentExtractorPath: nonExecutableExtractor.Name(),
}, logging.DefaultLogger, podstatus.NewConsul(statusstoretest.NewFake(), kp.PreparerPodStatusNamespace))
if reporter != nil || err == nil {
t.Errorf("Should have gotten a nil reporter with non-executable environemnt_extractor_path")
}
// now make it executable
executableExtractor := nonExecutableExtractor
err = executableExtractor.Chmod(0777)
if err != nil {
t.Fatalf("Could not make environment extractor path executable: %s", err)
}
reporter, err = New(ReporterConfig{
SQLiteDatabasePath: "foo",
EnvironmentExtractorPath: executableExtractor.Name(),
}, logging.DefaultLogger, podstatus.NewConsul(statusstoretest.NewFake(), kp.PreparerPodStatusNamespace))
if err != nil {
t.Errorf("Unexpected error calling New(): %s", err)
}
}
// Instantiates a preparer with a podStore and podStatusStore backed by a fake
// consul KV implementation. The podStore field is necessary for acquiring the
// "intent" manifest for uuid pods, while the podStatusStore is necessary for
// getting the reality manifest
func testResultSetPreparer() *Preparer {
fakeStatusStore := statusstoretest.NewFake()
return &Preparer{
podStatusStore: podstatus.NewConsul(fakeStatusStore, kp.PreparerPodStatusNamespace),
podStore: podstore.NewConsul(consulutil.NewFakeClient().KV()),
}
}
func NewConsulStore(client consulutil.ConsulClient) *consulStore {
statusStore := statusstore.NewConsul(client)
podStatusStore := podstatus.NewConsul(statusStore, PreparerPodStatusNamespace)
podStore := podstore.NewConsul(client.KV())
return &consulStore{
client: client,
podStore: podStore,
podStatusStore: podStatusStore,
}
}
func main() {
// Parse custom flags + standard Consul routing options
_, opts, _ := flags.ParseWithConsulOptions()
client := kp.NewConsulClient(opts)
podStore := kp_podstore.NewConsul(client.KV())
podStatusStore := podstatus.NewConsul(statusstore.NewConsul(client), kp.PreparerPodStatusNamespace)
logger := log.New(os.Stderr, "", 0)
port := getPort(logger)
lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", port))
if err != nil {
logger.Fatalf("failed to listen: %v", err)
}
s := grpc.NewServer()
podstore_protos.RegisterP2PodStoreServer(s, podstore.NewServer(podStore, podStatusStore))
if err := s.Serve(lis); err != nil {
logger.Fatalf("failed to serve: %v", err)
}
}
func startReporter(t *testing.T, tempDir string) (string, chan struct{}, podstatus.Store) {
dbPath := filepath.Join(tempDir, "finishes.db")
// The extractor doesn't need to do anything because we'll mimic its behavior in the test,
// but it needs to exist and be executable in order to pass constructor validation
extractor, err := os.Create(filepath.Join(tempDir, "env_extractor"))
if err != nil {
t.Fatalf("Could not create env extractor file: %s", err)
}
// make it executable
err = extractor.Chmod(0777)
if err != nil {
t.Fatalf("could not make env extractor executable: %s", err)
}
config := ReporterConfig{
SQLiteDatabasePath: dbPath,
// This setting only matters for generating the FinishExec in the preparer,
// it won't affect the tests in this file
EnvironmentExtractorPath: extractor.Name(),
WorkspaceDirPath: tempDir,
PollInterval: 1 * time.Millisecond,
}
store := podstatus.NewConsul(statusstoretest.NewFake(), kp.PreparerPodStatusNamespace)
reporter, err := New(config, logging.DefaultLogger, store)
if err != nil {
t.Fatalf("Error creating reporter: %s", err)
}
quitCh := make(chan struct{})
err = reporter.Run(quitCh)
if err != nil {
t.Fatalf("Unable to start reporter: %s", err)
}
return dbPath, quitCh, store
}
func setupServerWithFakePodStatusStore() (podstatus.Store, store) {
fakePodStatusStore := podstatus.NewConsul(statusstoretest.NewFake(), kp.PreparerPodStatusNamespace)
return fakePodStatusStore, store{
podStatusStore: fakePodStatusStore,
}
}
func New(preparerConfig *PreparerConfig, logger logging.Logger) (*Preparer, error) {
addHooks(preparerConfig, logger)
if preparerConfig.ConsulAddress == "" {
return nil, util.Errorf("No Consul address given to the preparer")
}
if preparerConfig.PodRoot == "" {
return nil, util.Errorf("No pod root given to the preparer")
}
if preparerConfig.LogLevel != "" {
lv, err := logrus.ParseLevel(preparerConfig.LogLevel)
if err != nil {
return nil, util.Errorf("Received invalid log level %q", preparerConfig.LogLevel)
}
logger.Logger.Level = lv
}
authPolicy, err := getDeployerAuth(preparerConfig)
if err != nil {
return nil, err
}
artifactVerifier, err := getArtifactVerifier(preparerConfig, &logger)
if err != nil {
return nil, err
}
artifactRegistry, err := getArtifactRegistry(preparerConfig)
if err != nil {
return nil, err
}
client, err := preparerConfig.GetConsulClient()
if err != nil {
return nil, err
}
statusStore := statusstore.NewConsul(client)
podStatusStore := podstatus.NewConsul(statusStore, kp.PreparerPodStatusNamespace)
podStore := podstore.NewConsul(client.KV())
store := kp.NewConsulStore(client)
maxLaunchableDiskUsage := launch.DefaultAllowableDiskUsage
if preparerConfig.MaxLaunchableDiskUsage != "" {
maxLaunchableDiskUsage, err = size.Parse(preparerConfig.MaxLaunchableDiskUsage)
if err != nil {
return nil, util.Errorf("Unparseable value for max_launchable_disk_usage %v, %v", preparerConfig.MaxLaunchableDiskUsage, err)
}
}
err = os.MkdirAll(preparerConfig.PodRoot, 0755)
if err != nil {
return nil, util.Errorf("Could not create preparer pod directory: %s", err)
}
var logExec []string
if len(preparerConfig.LogExec) > 0 {
logExec = preparerConfig.LogExec
} else {
logExec = runit.DefaultLogExec()
}
finishExec := pods.NopFinishExec
var podProcessReporter *podprocess.Reporter
if preparerConfig.PodProcessReporterConfig.FullyConfigured() {
podProcessReporterLogger := logger.SubLogger(logrus.Fields{
"component": "PodProcessReporter",
})
podProcessReporter, err = podprocess.New(preparerConfig.PodProcessReporterConfig, podProcessReporterLogger, podStatusStore)
if err != nil {
return nil, err
}
finishExec = preparerConfig.PodProcessReporterConfig.FinishExec()
}
var hooksManifest manifest.Manifest
var hooksPod *pods.Pod
if preparerConfig.HooksManifest != NoHooksSentinelValue {
if preparerConfig.HooksManifest == "" {
return nil, util.Errorf("Most provide a hooks_manifest or sentinel value %q to indicate that there are no hooks", NoHooksSentinelValue)
}
hooksManifest, err = manifest.FromBytes([]byte(preparerConfig.HooksManifest))
if err != nil {
return nil, util.Errorf("Could not parse configured hooks manifest: %s", err)
}
hooksPodFactory := pods.NewHookFactory(filepath.Join(preparerConfig.PodRoot, "hooks"), preparerConfig.NodeName)
hooksPod = hooksPodFactory.NewHookPod(hooksManifest.ID())
}
return &Preparer{
node: preparerConfig.NodeName,
store: store,
hooks: hooks.Hooks(preparerConfig.HooksDirectory, preparerConfig.PodRoot, &logger),
podStatusStore: podStatusStore,
podStore: podStore,
Logger: logger,
podFactory: pods.NewFactory(preparerConfig.PodRoot, preparerConfig.NodeName),
authPolicy: authPolicy,
maxLaunchableDiskUsage: maxLaunchableDiskUsage,
finishExec: finishExec,
logExec: logExec,
logBridgeBlacklist: preparerConfig.LogBridgeBlacklist,
artifactVerifier: artifactVerifier,
artifactRegistry: artifactRegistry,
PodProcessReporter: podProcessReporter,
hooksManifest: hooksManifest,
hooksPod: hooksPod,
hooksExecDir: preparerConfig.HooksDirectory,
}, nil
}
func verifyProcessExit(errCh chan error, tempDir string, logger logging.Logger) {
defer close(errCh)
// Schedule a uuid pod
podUniqueKey, err := createHelloUUIDPod(tempDir, 43772, logger)
if err != nil {
errCh <- fmt.Errorf("Could not schedule UUID hello pod: %s", err)
return
}
logger = logger.SubLogger(logrus.Fields{
"pod_unique_key": podUniqueKey,
})
logger.Infoln("Scheduled hello instance on port 43772")
err = verifyHelloRunning(podUniqueKey, logger)
if err != nil {
errCh <- fmt.Errorf("Couldn't get hello running as a uuid pod: %s", err)
return
}
logger.Infoln("Hello instance launched")
time.Sleep(3 * time.Second)
logger.Infoln("Waiting for hello instance to listen on 43772")
// now wait for the hello server to start running
timeout := time.After(30 * time.Second)
for {
resp, err := http.Get("http://localhost:43772/")
if err == nil {
resp.Body.Close()
break
}
select {
case <-timeout:
errCh <- fmt.Errorf("Hello didn't come up listening on 43772: %s", err)
return
default:
}
time.Sleep(1 * time.Second)
}
exitCode := rand.Intn(100) + 1
logger.Infof("Causing hello on 43772 to exit with status %d", exitCode)
// Make an http request to hello to make it exit with exitCode. We expect the http request to fail due
// to the server exiting, so don't check for http errors.
_, err = http.Get(fmt.Sprintf("http://localhost:43772/exit/%d", exitCode))
if err == nil {
// This is bad, it means the hello server didn't die and kill our request
// in the middle
errCh <- util.Errorf("Couldn't kill hello server with http request")
return
}
urlError, ok := err.(*url.Error)
if ok && urlError.Err == io.EOF {
// This is good, it means the server died
} else {
errCh <- fmt.Errorf("Couldn't tell hello to die over http: %s", err)
return
}
logger.Infoln("Checking for exit code in SQL database")
finishService, err := podprocess.NewSQLiteFinishService(sqliteFinishDatabasePath, logging.DefaultLogger)
if err != nil {
errCh <- err
return
}
var finishResult podprocess.FinishOutput
timeout = time.After(30 * time.Second)
for {
finishResult, err = finishService.LastFinishForPodUniqueKey(podUniqueKey)
if err == nil {
break
}
select {
case <-timeout:
// Try to manually run the finish script in order to make debugging the test failure easier
output, err := exec.Command("sudo", fmt.Sprintf("/var/service/hello-%s__hello__launch/finish", podUniqueKey), "1", "2").CombinedOutput()
if err != nil {
logger.WithError(err).Infoln("DEBUG: Debug attempt to run finish script failed")
}
logger.Infof("DEBUG: Output of direct execution of finish script: %s", string(output))
errCh <- fmt.Errorf("Did not find a finish row by the deadline: %s", err)
return
default:
}
}
if finishResult.PodUniqueKey != podUniqueKey {
errCh <- fmt.Errorf("Expected finish result for '%s' but it was for '%s'", podUniqueKey, finishResult.PodUniqueKey)
return
}
if finishResult.ExitCode != exitCode {
errCh <- fmt.Errorf("Exit code for '%s' in the sqlite database was expected to be %d but was %d", podUniqueKey, exitCode, finishResult.ExitCode)
return
}
logger.Infoln("Checking for exit code in consul")
timeout = time.After(30 * time.Second)
podStatusStore := podstatus.NewConsul(statusstore.NewConsul(kp.NewConsulClient(kp.Options{})), kp.PreparerPodStatusNamespace)
for {
podStatus, _, err := podStatusStore.Get(podUniqueKey)
if err != nil {
errCh <- err
return
}
found := false
for _, processStatus := range podStatus.ProcessStatuses {
if processStatus.LaunchableID == "hello" && processStatus.EntryPoint == "launch" {
found = true
if processStatus.LastExit == nil {
errCh <- fmt.Errorf("Found no last exit in consul pod status for %s", podUniqueKey)
return
}
if processStatus.LastExit.ExitCode != exitCode {
errCh <- fmt.Errorf("Exit code for '%s' in consul was expected to be %d but was %d", podUniqueKey, exitCode, finishResult.ExitCode)
return
}
}
}
if found {
logger.Infoln("Successful!")
break
}
select {
case <-timeout:
errCh <- fmt.Errorf("There was no pod process for hello/launch for %s in consul", podUniqueKey)
return
default:
}
}
}