func main() {
InitLogger(logrus.DebugLevel.String(), "yws1")
serfConfig := serf.DefaultConfig()
serfConfig.NodeName = "yws1"
serfConfig.MemberlistConfig = memberlist.DefaultLANConfig()
serfConfig.MemberlistConfig.BindAddr = "127.0.0.1"
serfConfig.MemberlistConfig.BindPort = 7373
serfConfig.MemberlistConfig.AdvertiseAddr = "127.0.0.1"
serfConfig.MemberlistConfig.AdvertisePort = 5000
eventCh := make(chan serf.Event, 64)
serfConfig.EventCh = eventCh
serfConfig.LogOutput = ioutil.Discard
serfConfig.MemberlistConfig.LogOutput = ioutil.Discard
s, err := serf.Create(serfConfig)
if err != nil {
fmt.Println(err)
}
go func() {
serfShutdownCh := s.ShutdownCh()
log.Info("agent: Listen for events")
for {
select {
case e := <-eventCh:
log.WithFields(logrus.Fields{
"event": e.String(),
}).Debug("agent: Received event")
// Log all member events
if failed, ok := e.(serf.MemberEvent); ok {
for _, member := range failed.Members {
log.WithFields(logrus.Fields{
"node": "yang",
"member": member.Name,
"event": e.EventType(),
}).Debug("agent: Member event")
}
}
if e.EventType() == serf.EventQuery {
query := e.(*serf.Query)
fmt.Println(query)
}
case <-serfShutdownCh:
log.Warn("agent: Serf shutdown detected, quitting")
return
}
}
}()
fmt.Println(s)
wait := make(chan int)
<-wait
}
func (d *driver) serfInit() error {
var err error
config := serf.DefaultConfig()
config.Init()
config.MemberlistConfig.BindAddr = d.bindAddress
d.eventCh = make(chan serf.Event, 4)
config.EventCh = d.eventCh
config.UserCoalescePeriod = 1 * time.Second
config.UserQuiescentPeriod = 50 * time.Millisecond
config.LogOutput = &logWriter{}
config.MemberlistConfig.LogOutput = config.LogOutput
s, err := serf.Create(config)
if err != nil {
return fmt.Errorf("failed to create cluster node: %v", err)
}
defer func() {
if err != nil {
s.Shutdown()
}
}()
d.serfInstance = s
d.notifyCh = make(chan ovNotify)
d.exitCh = make(chan chan struct{})
go d.startSerfLoop(d.eventCh, d.notifyCh, d.exitCh)
return nil
}
// setupSerf is used to setup and initialize a Serf
func (s *Server) setupSerf(conf *serf.Config, ch chan serf.Event, path string, wan bool) (*serf.Serf, error) {
addr := s.rpcListener.Addr().(*net.TCPAddr)
conf.Init()
if wan {
conf.NodeName = fmt.Sprintf("%s.%s", s.config.NodeName, s.config.Datacenter)
} else {
conf.NodeName = s.config.NodeName
}
conf.Tags["role"] = "consul"
conf.Tags["dc"] = s.config.Datacenter
conf.Tags["vsn"] = fmt.Sprintf("%d", s.config.ProtocolVersion)
conf.Tags["vsn_min"] = fmt.Sprintf("%d", ProtocolVersionMin)
conf.Tags["vsn_max"] = fmt.Sprintf("%d", ProtocolVersionMax)
conf.Tags["port"] = fmt.Sprintf("%d", addr.Port)
if s.config.Bootstrap {
conf.Tags["bootstrap"] = "1"
}
conf.MemberlistConfig.LogOutput = s.config.LogOutput
conf.LogOutput = s.config.LogOutput
conf.EventCh = ch
conf.SnapshotPath = filepath.Join(s.config.DataDir, path)
conf.ProtocolVersion = protocolVersionMap[s.config.ProtocolVersion]
// Until Consul supports this fully, we disable automatic resolution.
// When enabled, the Serf gossip may just turn off if we are the minority
// node which is rather unexpected.
conf.EnableNameConflictResolution = false
if err := ensurePath(conf.SnapshotPath, false); err != nil {
return nil, err
}
return serf.Create(conf)
}
// setupSerf is used to setup and initialize a Serf
func (s *Server) setupSerf(conf *serf.Config, ch chan serf.Event, path string) (*serf.Serf, error) {
conf.Init()
conf.NodeName = fmt.Sprintf("%s.%s", s.config.NodeName, s.config.Region)
conf.Tags["role"] = "nomad"
conf.Tags["region"] = s.config.Region
conf.Tags["dc"] = s.config.Datacenter
conf.Tags["vsn"] = fmt.Sprintf("%d", structs.ApiMajorVersion)
conf.Tags["mvn"] = fmt.Sprintf("%d", structs.ApiMinorVersion)
conf.Tags["build"] = s.config.Build
conf.Tags["port"] = fmt.Sprintf("%d", s.rpcAdvertise.(*net.TCPAddr).Port)
if s.config.Bootstrap || (s.config.DevMode && !s.config.DevDisableBootstrap) {
conf.Tags["bootstrap"] = "1"
}
if s.config.BootstrapExpect != 0 {
conf.Tags["expect"] = fmt.Sprintf("%d", s.config.BootstrapExpect)
}
conf.MemberlistConfig.LogOutput = s.config.LogOutput
conf.LogOutput = s.config.LogOutput
conf.EventCh = ch
if !s.config.DevMode {
conf.SnapshotPath = filepath.Join(s.config.DataDir, path)
if err := ensurePath(conf.SnapshotPath, false); err != nil {
return nil, err
}
}
conf.ProtocolVersion = protocolVersionMap[s.config.ProtocolVersion]
conf.RejoinAfterLeave = true
conf.Merge = &serfMergeDelegate{}
// Until Nomad supports this fully, we disable automatic resolution.
// When enabled, the Serf gossip may just turn off if we are the minority
// node which is rather unexpected.
conf.EnableNameConflictResolution = false
return serf.Create(conf)
}
func (a *Agent) Start() error {
conf := serf.DefaultConfig()
conf.Init()
conf.Tags["role"] = "agent"
conf.Tags["info"] = a.getInfo()
bindAddr, err := a.config.GetIpByInterface()
if err != nil {
log.Fatal(err)
}
conf.NodeName = a.config.Name
conf.MemberlistConfig.BindAddr = bindAddr
conf.EventCh = a.eventCh
serf, err := serf.Create(conf)
if err != nil {
return err
}
a.serf = serf
go a.handleEvents()
return nil
}
func serfInit() *serf.Serf {
conf := serf.DefaultConfig()
serfin, err := serf.Create(conf)
if err != nil {
log.Fatal(err)
}
return serfin
}
// Start starts the agent, kicking off any goroutines to handle various
// aspects of the agent.
func (a *Agent) Start() error {
a.once.Do(a.init)
a.lock.Lock()
defer a.lock.Unlock()
a.logger.Println("[INFO] Serf agent starting")
// Setup logging a bit
a.SerfConfig.MemberlistConfig.LogOutput = a.LogOutput
a.SerfConfig.LogOutput = a.LogOutput
eventCh := make(chan serf.Event, 64)
a.SerfConfig.EventCh = eventCh
var err error
a.serf, err = serf.Create(a.SerfConfig)
if err != nil {
return fmt.Errorf("Error creating Serf: %s", err)
}
a.rpcListener, err = net.Listen("tcp", a.RPCAddr)
if err != nil {
return fmt.Errorf("Error starting RPC listener: %s", err)
}
rpcServer := rpc.NewServer()
err = registerEndpoint(rpcServer, a)
if err != nil {
return fmt.Errorf("Error starting RPC server: %s", err)
}
go func(l net.Listener) {
for {
conn, err := l.Accept()
if err != nil {
a.logger.Printf("[ERR] RPC accept error: %s", err)
return
}
go rpcServer.ServeConn(conn)
}
}(a.rpcListener)
shutdownCh := make(chan struct{})
// Only listen for events if we care about events.
go a.eventLoop(a.EventHandler, eventCh, shutdownCh)
a.shutdownCh = shutdownCh
a.state = AgentRunning
a.logger.Println("[INFO] Serf agent started")
return nil
}
func (a *Agent) Start() error {
log.Info("agent: Serf agent starting")
// Create serf first
serf, err := serf.Create(a.conf)
if err != nil {
return fmt.Errorf("Error creating Serf: %s", err)
}
a.serf = serf
return nil
}
// Start is used to initiate the event listeners. It is seperate from
// create so that there isn't a race condition between creating the
// agent and registering handlers
func (a *Agent) Start() error {
a.logger.Printf("[INFO] agent: Serf agent starting")
// Create serf first
serf, err := serf.Create(a.conf)
if err != nil {
return fmt.Errorf("Error creating Serf: %s", err)
}
a.serf = serf
// Start event loop
go a.eventLoop()
return nil
}
func (s *Server) setupSerf() (*serf.Serf, error) {
conf := serf.DefaultConfig()
// Generate NodeName if missing
id, err := uuid.UUID4()
if err != nil {
return nil, err
}
// Get SSH server port
port := s.sshServer.Addr.Port
// Initialize serf
conf.Init()
// s.logger.Info("local port", "port", conf.MemberlistConfig.BindPort)
conf.NodeName = s.config.NodeName
conf.MemberlistConfig.BindAddr = s.config.GossipBindAddr
conf.MemberlistConfig.BindPort = s.config.GossipBindPort
conf.MemberlistConfig.AdvertiseAddr = s.config.GossipAdvertiseAddr
conf.MemberlistConfig.AdvertisePort = s.config.GossipAdvertisePort
s.logger.Info("Gossip", "BindAddr", conf.MemberlistConfig.BindAddr, "BindPort", conf.MemberlistConfig.BindPort, "AdvertiseAddr", conf.MemberlistConfig.AdvertiseAddr, "AdvertisePort", conf.MemberlistConfig.AdvertisePort)
conf.Tags["id"] = id
conf.Tags["role"] = "kappa-server"
conf.Tags["cluster"] = s.config.ClusterName
conf.Tags["build"] = s.config.Build
conf.Tags["port"] = fmt.Sprintf("%d", port)
if s.config.Bootstrap {
conf.Tags["bootstrap"] = "1"
}
if s.config.BootstrapExpect != 0 {
conf.Tags["expect"] = fmt.Sprintf("%d", s.config.BootstrapExpect)
}
conf.MemberlistConfig.LogOutput = s.config.LogOutput
conf.LogOutput = s.config.LogOutput
conf.EventCh = s.serfEventCh
conf.SnapshotPath = filepath.Join(s.config.DataPath, serfSnapshot)
conf.ProtocolVersion = conf.ProtocolVersion
conf.RejoinAfterLeave = true
conf.EnableNameConflictResolution = false
conf.Merge = &mergeDelegate{name: s.config.ClusterName}
if err := ensurePath(conf.SnapshotPath, false); err != nil {
return nil, err
}
return serf.Create(conf)
}
// setupSerf sets up serf and provides a handle on its events
func (s *Server) setupSerf() (*serf.Serf, error) {
// initializes the config (contains maps)
s.config.SerfConfig.Init()
s.config.SerfConfig.EventCh = s.eventChLAN
if s.config.Leader {
s.config.SerfConfig.Tags["role"] = "leader"
} else {
s.config.SerfConfig.Tags["role"] = "follower"
}
serfLAN, err := serf.Create(s.config.SerfConfig)
if err != nil {
return nil, err
}
return serfLAN, nil
}
func (d *driver) serfInit() error {
var err error
config := serf.DefaultConfig()
config.Init()
if d.ifaceName != "" {
bindAddr, err := getBindAddr(d.ifaceName)
if err != nil {
return fmt.Errorf("getBindAddr error: %v", err)
}
config.MemberlistConfig.BindAddr = bindAddr
}
d.eventCh = make(chan serf.Event, 4)
config.EventCh = d.eventCh
config.UserCoalescePeriod = 1 * time.Second
config.UserQuiescentPeriod = 50 * time.Millisecond
config.LogOutput = logrus.StandardLogger().Out
s, err := serf.Create(config)
if err != nil {
return fmt.Errorf("failed to create cluster node: %v", err)
}
defer func() {
if err != nil {
s.Shutdown()
}
}()
if d.neighIP != "" {
if _, err = s.Join([]string{d.neighIP}, false); err != nil {
return fmt.Errorf("Failed to join the cluster at neigh IP %s: %v",
d.neighIP, err)
}
}
d.serfInstance = s
d.notifyCh = make(chan ovNotify)
d.exitCh = make(chan chan struct{})
go d.startSerfLoop(d.eventCh, d.notifyCh, d.exitCh)
return nil
}
// setupSerf is used to setup and initialize a Serf
func (c *Client) setupSerf(conf *serf.Config, ch chan serf.Event, path string) (*serf.Serf, error) {
conf.Init()
conf.NodeName = c.config.NodeName
conf.Tags["role"] = "node"
conf.Tags["dc"] = c.config.Datacenter
conf.Tags["vsn"] = fmt.Sprintf("%d", c.config.ProtocolVersion)
conf.Tags["vsn_min"] = fmt.Sprintf("%d", ProtocolVersionMin)
conf.Tags["vsn_max"] = fmt.Sprintf("%d", ProtocolVersionMax)
conf.MemberlistConfig.LogOutput = c.config.LogOutput
conf.LogOutput = c.config.LogOutput
conf.EventCh = ch
conf.SnapshotPath = filepath.Join(c.config.DataDir, path)
conf.ProtocolVersion = protocolVersionMap[c.config.ProtocolVersion]
if err := ensurePath(conf.SnapshotPath, false); err != nil {
return nil, err
}
return serf.Create(conf)
}
// initSerf initializes serf and advertizes this nodes RPC port to other nodes.
func (n *Node) initSerf(config *serf.Config) error {
config.NodeName = n.name
config.EventCh = n.eventCh
// Advertize the RPC port
tags := map[string]string{
externalRpcTag: strconv.Itoa(n.port),
}
config.Tags = tags
// Start serf for discovery
s, err := serf.Create(config)
if err != nil {
return err
}
n.serf = s
return nil
}
// NewCluster returns a new cluster with the given config
func NewCluster(config Config) (*Cluster, error) {
logger := logrus.New()
logger.Out = config.LogOutput
logger.Level = logrus.Level(config.LogLevel)
if config.SerfConfig == nil {
return nil, fmt.Errorf("Config.SerfConfig cannot be nil")
}
if config.SerfConfig.EventCh != nil {
return nil, fmt.Errorf("SerfConfig.EventCh must be nil (try using Config.SerfEvents instead)")
}
memberMap := make(map[string]*serf.Member)
memberMutex := sync.Mutex{}
ring := &ring{members: make([]*serf.Member, 0, 0)}
serfEvents := make(chan serf.Event, 256)
config.SerfConfig.EventCh = serfEvents
nodeSerf, err := serf.Create(config.SerfConfig)
if err != nil {
return nil, fmt.Errorf("Unable to create serf: %v", err)
}
exit := make(chan bool)
cluster := &Cluster{
exit: exit,
config: config,
memberMap: memberMap,
ring: ring,
memberMutex: memberMutex,
serfEvents: serfEvents,
Serf: nodeSerf,
logger: logger,
}
return cluster, nil
}
// setupSerf is used to setup and initialize a Serf
func (s *Server) setupSerf(conf *serf.Config, ch chan serf.Event, path string) (*serf.Serf, error) {
addr := s.rpcListener.Addr().(*net.TCPAddr)
conf.Init()
conf.NodeName = s.config.NodeName
conf.Tags["role"] = "consul"
conf.Tags["dc"] = s.config.Datacenter
conf.Tags["vsn"] = fmt.Sprintf("%d", s.config.ProtocolVersion)
conf.Tags["vsn_min"] = fmt.Sprintf("%d", ProtocolVersionMin)
conf.Tags["vsn_max"] = fmt.Sprintf("%d", ProtocolVersionMax)
conf.Tags["port"] = fmt.Sprintf("%d", addr.Port)
if s.config.Bootstrap {
conf.Tags["bootstrap"] = "1"
}
conf.MemberlistConfig.LogOutput = s.config.LogOutput
conf.LogOutput = s.config.LogOutput
conf.EventCh = ch
conf.SnapshotPath = filepath.Join(s.config.DataDir, path)
conf.ProtocolVersion = protocolVersionMap[s.config.ProtocolVersion]
if err := ensurePath(conf.SnapshotPath, false); err != nil {
return nil, err
}
return serf.Create(conf)
}
func (c *cerebrum) setupSerf() (*serf.Serf, error) {
// Get serf config
conf := serf.DefaultConfig()
// Initialize serf
conf.Init()
conf.NodeName = c.config.NodeName
conf.MemberlistConfig.BindAddr = c.config.GossipBindAddr
conf.MemberlistConfig.BindPort = c.config.GossipBindPort
conf.MemberlistConfig.AdvertiseAddr = c.config.GossipAdvertiseAddr
conf.MemberlistConfig.AdvertisePort = c.config.GossipAdvertisePort
c.logger.Info("Gossip",
"BindAddr", conf.MemberlistConfig.BindAddr,
"BindPort", conf.MemberlistConfig.BindPort,
"AdvertiseAddr", conf.MemberlistConfig.AdvertiseAddr,
"AdvertisePort", conf.MemberlistConfig.AdvertisePort)
conf.Tags["id"] = c.config.NodeID
conf.Tags["role"] = "cerebrum-server"
conf.Tags["dc"] = c.config.DataCenter
// conf.Tags["port"] = fmt.Sprintf("%d", port)
conf.MemberlistConfig.LogOutput = c.config.LogOutput
conf.LogOutput = c.config.LogOutput
conf.EventCh = c.serfEventCh
conf.SnapshotPath = filepath.Join(c.config.DataPath, SerfSnapshotDir)
conf.ProtocolVersion = conf.ProtocolVersion
conf.RejoinAfterLeave = true
conf.EnableNameConflictResolution = false
conf.Merge = &mergeDelegate{c.logger}
if err := os.MkdirAll(conf.SnapshotPath, 0755); err != nil {
return nil, err
}
return serf.Create(conf)
}
// setupAgent is used to create the agent we use
func (a *AgentCommand) setupSerf(config *Config) *serf.Serf {
bindIP, bindPort, err := config.AddrParts(config.BindAddr)
if err != nil {
a.Ui.Error(fmt.Sprintf("Invalid bind address: %s", err))
return nil
}
// Check if we have an interface
if iface, _ := config.NetworkInterface(); iface != nil {
addrs, err := iface.Addrs()
if err != nil {
a.Ui.Error(fmt.Sprintf("Failed to get interface addresses: %s", err))
return nil
}
if len(addrs) == 0 {
a.Ui.Error(fmt.Sprintf("Interface '%s' has no addresses", config.Interface))
return nil
}
// If there is no bind IP, pick an address
if bindIP == "0.0.0.0" {
found := false
for _, ad := range addrs {
var addrIP net.IP
if runtime.GOOS == "windows" {
// Waiting for https://github.com/golang/go/issues/5395 to use IPNet only
addr, ok := ad.(*net.IPAddr)
if !ok {
continue
}
addrIP = addr.IP
} else {
addr, ok := ad.(*net.IPNet)
if !ok {
continue
}
addrIP = addr.IP
}
// Skip self-assigned IPs
if addrIP.IsLinkLocalUnicast() {
continue
}
// Found an IP
found = true
bindIP = addrIP.String()
a.Ui.Output(fmt.Sprintf("Using interface '%s' address '%s'",
config.Interface, bindIP))
// Update the configuration
bindAddr := &net.TCPAddr{
IP: net.ParseIP(bindIP),
Port: bindPort,
}
config.BindAddr = bindAddr.String()
break
}
if !found {
a.Ui.Error(fmt.Sprintf("Failed to find usable address for interface '%s'", config.Interface))
return nil
}
} else {
// If there is a bind IP, ensure it is available
found := false
for _, a := range addrs {
addr, ok := a.(*net.IPNet)
if !ok {
continue
}
if addr.IP.String() == bindIP {
found = true
break
}
}
if !found {
a.Ui.Error(fmt.Sprintf("Interface '%s' has no '%s' address",
config.Interface, bindIP))
return nil
}
}
}
var advertiseIP string
var advertisePort int
if config.AdvertiseAddr != "" {
advertiseIP, advertisePort, err = config.AddrParts(config.AdvertiseAddr)
if err != nil {
a.Ui.Error(fmt.Sprintf("Invalid advertise address: %s", err))
return nil
}
}
encryptKey, err := config.EncryptBytes()
if err != nil {
a.Ui.Error(fmt.Sprintf("Invalid encryption key: %s", err))
return nil
}
serfConfig := serf.DefaultConfig()
switch config.Profile {
case "lan":
serfConfig.MemberlistConfig = memberlist.DefaultLANConfig()
case "wan":
serfConfig.MemberlistConfig = memberlist.DefaultWANConfig()
case "local":
serfConfig.MemberlistConfig = memberlist.DefaultLocalConfig()
default:
a.Ui.Error(fmt.Sprintf("Unknown profile: %s", config.Profile))
return nil
}
serfConfig.MemberlistConfig.BindAddr = bindIP
serfConfig.MemberlistConfig.BindPort = bindPort
serfConfig.MemberlistConfig.AdvertiseAddr = advertiseIP
serfConfig.MemberlistConfig.AdvertisePort = advertisePort
serfConfig.MemberlistConfig.SecretKey = encryptKey
serfConfig.NodeName = config.NodeName
serfConfig.Tags = config.Tags
serfConfig.SnapshotPath = config.SnapshotPath
serfConfig.CoalescePeriod = 3 * time.Second
serfConfig.QuiescentPeriod = time.Second
serfConfig.UserCoalescePeriod = 3 * time.Second
serfConfig.UserQuiescentPeriod = time.Second
if config.ReconnectInterval != 0 {
serfConfig.ReconnectInterval = config.ReconnectInterval
}
if config.ReconnectTimeout != 0 {
serfConfig.ReconnectTimeout = config.ReconnectTimeout
}
if config.TombstoneTimeout != 0 {
serfConfig.TombstoneTimeout = config.TombstoneTimeout
}
serfConfig.EnableNameConflictResolution = !config.DisableNameResolution
if config.KeyringFile != "" {
serfConfig.KeyringFile = config.KeyringFile
}
serfConfig.RejoinAfterLeave = config.RejoinAfterLeave
// Create a channel to listen for events from Serf
a.eventCh = make(chan serf.Event, 64)
serfConfig.EventCh = a.eventCh
// Start Serf
a.Ui.Output("Starting Serf agent...")
log.Info("agent: Serf agent starting")
serfConfig.LogOutput = ioutil.Discard
serfConfig.MemberlistConfig.LogOutput = ioutil.Discard
// Create serf first
serf, err := serf.Create(serfConfig)
if err != nil {
a.Ui.Error(err.Error())
log.Error(err)
return nil
}
return serf
}