func testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubResponse, crunchCmd string, expected string) { apiStubResponses["/arvados/v1/api_client_authorizations/current"] = arvadostest.StubResponse{200, string(`{"uuid": "` + arvadostest.Dispatch1AuthUUID + `", "api_token": "xyz"}`)} apiStub := arvadostest.ServerStub{apiStubResponses} api := httptest.NewServer(&apiStub) defer api.Close() arv := arvadosclient.ArvadosClient{ Scheme: "http", ApiServer: api.URL[7:], ApiToken: "abc123", Client: &http.Client{Transport: &http.Transport{}}, Retries: 0, } buf := bytes.NewBuffer(nil) log.SetOutput(io.MultiWriter(buf, os.Stderr)) defer log.SetOutput(os.Stderr) *crunchRunCommand = crunchCmd doneProcessing := make(chan struct{}) dispatcher := dispatch.Dispatcher{ Arv: arv, PollInterval: time.Duration(1) * time.Second, RunContainer: func(dispatcher *dispatch.Dispatcher, container dispatch.Container, status chan dispatch.Container) { run(dispatcher, container, status) doneProcessing <- struct{}{} }, DoneProcessing: doneProcessing} startCmd = func(container dispatch.Container, cmd *exec.Cmd) error { dispatcher.UpdateState(container.UUID, "Running") dispatcher.UpdateState(container.UUID, "Complete") return cmd.Start() } go func() { for i := 0; i < 80 && !strings.Contains(buf.String(), expected); i++ { time.Sleep(100 * time.Millisecond) } dispatcher.DoneProcessing <- struct{}{} }() err := dispatcher.RunDispatcher() c.Assert(err, IsNil) // Wait for all running crunch jobs to complete / terminate waitGroup.Wait() c.Check(buf.String(), Matches, `(?ms).*`+expected+`.*`) }
func testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubResponse, crunchCmd string, expected string) { apiStub := arvadostest.ServerStub{apiStubResponses} api := httptest.NewServer(&apiStub) defer api.Close() arv := arvadosclient.ArvadosClient{ Scheme: "http", ApiServer: api.URL[7:], ApiToken: "abc123", Client: &http.Client{Transport: &http.Transport{}}, Retries: 0, } buf := bytes.NewBuffer(nil) log.SetOutput(io.MultiWriter(buf, os.Stderr)) defer log.SetOutput(os.Stderr) crunchRunCommand = &crunchCmd doneProcessing := make(chan struct{}) dispatcher := dispatch.Dispatcher{ Arv: arv, PollInterval: time.Duration(1) * time.Second, RunContainer: func(dispatcher *dispatch.Dispatcher, container dispatch.Container, status chan dispatch.Container) { go func() { time.Sleep(1 * time.Second) dispatcher.UpdateState(container.UUID, dispatch.Running) dispatcher.UpdateState(container.UUID, dispatch.Complete) }() run(dispatcher, container, status) doneProcessing <- struct{}{} }, DoneProcessing: doneProcessing} go func() { for i := 0; i < 80 && !strings.Contains(buf.String(), expected); i++ { time.Sleep(100 * time.Millisecond) } dispatcher.DoneProcessing <- struct{}{} }() err := dispatcher.RunDispatcher() c.Assert(err, IsNil) c.Check(buf.String(), Matches, `(?ms).*`+expected+`.*`) }
func doMain() error { flags := flag.NewFlagSet("crunch-dispatch-local", flag.ExitOnError) pollInterval := flags.Int( "poll-interval", 10, "Interval in seconds to poll for queued containers") crunchRunCommand = flags.String( "crunch-run-command", "/usr/bin/crunch-run", "Crunch command to run container") // Parse args; omit the first arg which is the command name flags.Parse(os.Args[1:]) runningCmds = make(map[string]*exec.Cmd) arv, err := arvadosclient.MakeArvadosClient() if err != nil { log.Printf("Error making Arvados client: %v", err) return err } arv.Retries = 25 dispatcher := dispatch.Dispatcher{ Arv: arv, RunContainer: run, PollInterval: time.Duration(*pollInterval) * time.Second, DoneProcessing: make(chan struct{})} err = dispatcher.RunDispatcher() if err != nil { return err } runningCmdsMutex.Lock() // Finished dispatching; interrupt any crunch jobs that are still running for _, cmd := range runningCmds { cmd.Process.Signal(os.Interrupt) } runningCmdsMutex.Unlock() // Wait for all running crunch jobs to complete / terminate waitGroup.Wait() return nil }
func (s *TestSuite) TestIntegration(c *C) { arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, IsNil) echo := "echo" crunchRunCommand = &echo doneProcessing := make(chan struct{}) dispatcher := dispatch.Dispatcher{ Arv: arv, PollInterval: time.Duration(1) * time.Second, RunContainer: func(dispatcher *dispatch.Dispatcher, container dispatch.Container, status chan dispatch.Container) { run(dispatcher, container, status) doneProcessing <- struct{}{} }, DoneProcessing: doneProcessing} startCmd = func(container dispatch.Container, cmd *exec.Cmd) error { dispatcher.UpdateState(container.UUID, "Running") dispatcher.UpdateState(container.UUID, "Complete") return cmd.Start() } err = dispatcher.RunDispatcher() c.Assert(err, IsNil) // Wait for all running crunch jobs to complete / terminate waitGroup.Wait() // There should be no queued containers now params := arvadosclient.Dict{ "filters": [][]string{[]string{"state", "=", "Queued"}}, } var containers dispatch.ContainerList err = arv.List("containers", params, &containers) c.Check(err, IsNil) c.Assert(len(containers.Items), Equals, 0) // Previously "Queued" container should now be in "Complete" state var container dispatch.Container err = arv.Get("containers", "zzzzz-dz642-queuedcontainer", nil, &container) c.Check(err, IsNil) c.Check(container.State, Equals, "Complete") }
func doMain() error { flags := flag.NewFlagSet("crunch-dispatch-slurm", flag.ExitOnError) pollInterval := flags.Int( "poll-interval", 10, "Interval in seconds to poll for queued containers") crunchRunCommand = flags.String( "crunch-run-command", "/usr/bin/crunch-run", "Crunch command to run container") // Parse args; omit the first arg which is the command name flags.Parse(os.Args[1:]) arv, err := arvadosclient.MakeArvadosClient() if err != nil { log.Printf("Error making Arvados client: %v", err) return err } arv.Retries = 25 squeueUpdater.StartMonitor(time.Duration(*pollInterval) * time.Second) defer squeueUpdater.Done() dispatcher := dispatch.Dispatcher{ Arv: arv, RunContainer: run, PollInterval: time.Duration(*pollInterval) * time.Second, DoneProcessing: make(chan struct{})} err = dispatcher.RunDispatcher() if err != nil { return err } return nil }
func (s *TestSuite) integrationTest(c *C, newSqueueCmd func() *exec.Cmd, sbatchCmdComps []string, runContainer func(*dispatch.Dispatcher, dispatch.Container)) dispatch.Container { arvadostest.ResetEnv() arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, IsNil) var sbatchCmdLine []string // Override sbatchCmd defer func(orig func(dispatch.Container) *exec.Cmd) { sbatchCmd = orig }(sbatchCmd) sbatchCmd = func(container dispatch.Container) *exec.Cmd { sbatchCmdLine = sbatchFunc(container).Args return exec.Command("sh") } // Override squeueCmd defer func(orig func() *exec.Cmd) { squeueCmd = orig }(squeueCmd) squeueCmd = newSqueueCmd // There should be no queued containers now params := arvadosclient.Dict{ "filters": [][]string{[]string{"state", "=", "Queued"}}, } var containers dispatch.ContainerList err = arv.List("containers", params, &containers) c.Check(err, IsNil) c.Check(len(containers.Items), Equals, 1) echo := "echo" crunchRunCommand = &echo doneProcessing := make(chan struct{}) dispatcher := dispatch.Dispatcher{ Arv: arv, PollInterval: time.Duration(1) * time.Second, RunContainer: func(dispatcher *dispatch.Dispatcher, container dispatch.Container, status chan dispatch.Container) { go runContainer(dispatcher, container) run(dispatcher, container, status) doneProcessing <- struct{}{} }, DoneProcessing: doneProcessing} squeueUpdater.StartMonitor(time.Duration(500) * time.Millisecond) err = dispatcher.RunDispatcher() c.Assert(err, IsNil) squeueUpdater.Done() c.Check(sbatchCmdLine, DeepEquals, sbatchCmdComps) // There should be no queued containers now err = arv.List("containers", params, &containers) c.Check(err, IsNil) c.Check(len(containers.Items), Equals, 0) // Previously "Queued" container should now be in "Complete" state var container dispatch.Container err = arv.Get("containers", "zzzzz-dz642-queuedcontainer", nil, &container) c.Check(err, IsNil) return container }