// listenForNodeChanges listens for changes to node status using change feeds. // This function will block until the query fails func (c *Cluster) listenForNodeChanges() error { // Start listening to changes from a random active node node, err := c.GetRandomNode() if err != nil { return err } q, err := newQuery( DB("rethinkdb").Table("server_status").Changes(), map[string]interface{}{}, c.opts, ) if err != nil { return fmt.Errorf("Error building query %s", err) } cursor, err := node.Query(q) if err != nil { return err } // Keep reading node status updates from changefeed var result struct { NewVal nodeStatus `gorethink:"new_val"` OldVal nodeStatus `gorethink:"old_val"` } for cursor.Next(&result) { addr := fmt.Sprintf("%s:%d", result.NewVal.Network.Hostname, result.NewVal.Network.ReqlPort) addr = strings.ToLower(addr) switch result.NewVal.Status { case "connected": // Connect to node using exponential backoff (give up after waiting 5s) // to give the node time to start-up. b := backoff.NewExponentialBackOff() b.MaxElapsedTime = time.Second * 5 backoff.Retry(func() error { node, err := c.connectNodeWithStatus(result.NewVal) if err == nil { if !c.nodeExists(node) { c.addNode(node) Log.WithFields(logrus.Fields{ "id": node.ID, "host": node.Host.String(), }).Debug("Connected to node") } } return err }, b) } } return cursor.Err() }
// discover attempts to find new nodes in the cluster using the current nodes func (c *Cluster) discover() { // Keep retrying with exponential backoff. b := backoff.NewExponentialBackOff() // Never finish retrying (max interval is still 60s) b.MaxElapsedTime = 0 // Keep trying to discover new nodes for { backoff.RetryNotify(func() error { // If no hosts try seeding nodes if len(c.GetNodes()) == 0 { c.connectNodes(c.getSeeds()) } return c.listenForNodeChanges() }, b, func(err error, wait time.Duration) { Log.Debugf("Error discovering hosts %s, waiting %s", err, wait) }) } }