func (r *Router) watchBackends() { log := r.logger.New("fn", "router.watchBackends", "router.id", r.ID) var events chan *router.StreamEvent var stream stream.Stream connect := func() (err error) { log.Info("connecting router event stream") events = make(chan *router.StreamEvent) opts := &router.StreamEventsOptions{ EventTypes: []router.EventType{ router.EventTypeBackendUp, router.EventTypeBackendDrained, }, } stream, err = r.client.StreamEvents(opts, events) if err != nil { log.Error("error connecting router event stream", "err", err) } return } // make initial connection for { if err := connect(); err == nil { defer stream.Close() break } select { case <-r.stop: return case <-time.After(100 * time.Millisecond): } } for { eventLoop: for { select { case event, ok := <-events: if !ok { break eventLoop } r.events <- &RouterEvent{ RouterID: r.ID, Type: event.Event, Backend: event.Backend, } case <-r.stop: return } } log.Warn("router event stream disconnected", "err", stream.Err()) // keep trying to reconnect, unless we are told to stop retryLoop: for { select { case <-r.stop: return default: } if err := connect(); err == nil { break retryLoop } time.Sleep(100 * time.Millisecond) } } }
func (c *serviceConn) watch(srv discoverd.Service, eventc <-chan *discoverd.Event, stream stream.Stream) { g := grohl.NewContext(grohl.Data{"at": "logmux_service_watch"}) var ( resetc = make(chan time.Time) reconc <-chan time.Time = resetc ) defer close(resetc) for { select { case event, ok := <-eventc: if !ok { c.hangup() return } g.Log(grohl.Data{"status": "event", "event": event.Kind.String()}) switch event.Kind { case discoverd.EventKindLeader: reconc = resetc if err := c.reset(); err != nil { g.Log(grohl.Data{"status": "error", "err": err.Error()}) } if err := c.connect(srv); err != nil { g.Log(grohl.Data{"status": "error", "err": err.Error()}) reconc = time.After(100 * time.Millisecond) } default: } case err := <-c.errc: g.Log(grohl.Data{"status": "write-error", "err": err.Error()}) reconc = resetc if err := c.reset(); err != nil { g.Log(grohl.Data{"status": "error", "err": err.Error()}) } if err := c.connect(srv); err != nil { g.Log(grohl.Data{"status": "error", "err": err.Error()}) reconc = time.After(100 * time.Millisecond) } case <-reconc: if err := c.connect(srv); err != nil { g.Log(grohl.Data{"status": "reconnect-error", "err": err.Error()}) reconc = time.After(100 * time.Millisecond) } case <-c.donec: if err := stream.Close(); err != nil { g.Log(grohl.Data{"status": "error", "err": err.Error()}) } if err := c.reset(); err != nil { g.Log(grohl.Data{"status": "error", "err": err.Error()}) } return case <-c.closec: if err := stream.Close(); err != nil { g.Log(grohl.Data{"status": "error", "err": err.Error()}) } c.hangup() return } } }
// StreamEventsTo streams all job events from the host to the given channel in // a goroutine, returning the current list of active jobs. func (h *Host) StreamEventsTo(ch chan *host.Event) (map[string]host.ActiveJob, error) { log := h.logger.New("fn", "StreamEventsTo", "host.id", h.ID) var events chan *host.Event var stream stream.Stream connect := func() (err error) { log.Info("connecting job event stream") events = make(chan *host.Event) stream, err = h.client.StreamEvents("all", events) if err != nil { log.Error("error connecting job event stream", "err", err) } return } if err := connect(); err != nil { return nil, err } log.Info("getting active jobs") jobs, err := h.client.ListJobs() if err != nil { log.Error("error getting active jobs", "err", err) return nil, err } log.Info(fmt.Sprintf("got %d active job(s) for host %s", len(jobs), h.ID)) go func() { defer stream.Close() defer close(h.done) for { eventLoop: for { select { case event, ok := <-events: if !ok { break eventLoop } ch <- event case <-h.stop: return } } log.Warn("job event stream disconnected", "err", stream.Err()) // keep trying to reconnect, unless we are told to stop retryLoop: for { select { case <-h.stop: return default: } if err := connect(); err == nil { break retryLoop } time.Sleep(100 * time.Millisecond) } } }() return jobs, nil }
func (s *SchedulerSuite) TestDeployController(t *c.C) { if testCluster == nil { t.Skip("cannot determine test cluster size") } // get the current controller release client := s.controllerClient(t) app, err := client.GetApp("controller") t.Assert(err, c.IsNil) release, err := client.GetAppRelease(app.ID) t.Assert(err, c.IsNil) // create a controller deployment release.ID = "" t.Assert(client.CreateRelease(release), c.IsNil) deployment, err := client.CreateDeployment(app.ID, release.ID) t.Assert(err, c.IsNil) // use a function to create the event stream as a new stream will be needed // after deploying the controller var events chan *ct.DeploymentEvent var eventStream stream.Stream connectStream := func() { events = make(chan *ct.DeploymentEvent) err := attempt.Strategy{ Total: 10 * time.Second, Delay: 500 * time.Millisecond, }.Run(func() (err error) { eventStream, err = client.StreamDeployment(deployment.ID, events) return }) t.Assert(err, c.IsNil) } connectStream() defer eventStream.Close() // wait for the deploy to complete (this doesn't wait for specific events // due to the fact that when the deployer deploys itself, some events will // not get sent) loop: for { select { case e, ok := <-events: if !ok { // reconnect the stream as it may of been closed // due to the controller being deployed debug(t, "reconnecting deployment event stream") connectStream() continue } debugf(t, "got deployment event: %s %s", e.JobType, e.JobState) switch e.Status { case "complete": break loop case "failed": t.Fatal("the deployment failed") } case <-time.After(60 * time.Second): t.Fatal("timed out waiting for the deploy to complete") } } // check the correct controller jobs are running hosts, err := s.clusterClient(t).ListHosts() t.Assert(err, c.IsNil) actual := make(map[string]map[string]int) for _, host := range hosts { for _, job := range host.Jobs { appID := job.Metadata["flynn-controller.app"] if appID != app.ID { continue } releaseID := job.Metadata["flynn-controller.release"] if _, ok := actual[releaseID]; !ok { actual[releaseID] = make(map[string]int) } typ := job.Metadata["flynn-controller.type"] actual[releaseID][typ]++ } } expected := map[string]map[string]int{release.ID: { "web": 2, "deployer": 2, "scheduler": testCluster.Size(), }} t.Assert(actual, c.DeepEquals, expected) }