func TestSnapshotter_forceCompact(t *testing.T) {
td, err := ioutil.TempDir("", "serf")
if err != nil {
t.Fatalf("err: %v", err)
}
defer os.RemoveAll(td)
// Set up a coordinate at a known location.
coordClient, err := coordinate.NewClient(coordinate.DefaultConfig())
if err != nil {
t.Fatalf("err: %v", err)
}
coord := coordinate.NewCoordinate(coordinate.DefaultConfig())
coord.Vec[0] = 123.4
coordClient.SetCoordinate(coord)
clock := new(LamportClock)
stopCh := make(chan struct{})
logger := log.New(os.Stderr, "", log.LstdFlags)
// Create a very low limit
inCh, snap, err := NewSnapshotter(td+"snap", 1024, false,
logger, clock, coordClient, nil, stopCh)
if err != nil {
t.Fatalf("err: %v", err)
}
// Write lots of user events
for i := 0; i < 1024; i++ {
ue := UserEvent{
LTime: LamportTime(i),
}
inCh <- ue
}
// Write lots of queries
for i := 0; i < 1024; i++ {
q := &Query{
LTime: LamportTime(i),
}
inCh <- q
}
// Wait for drain
for len(inCh) > 0 {
time.Sleep(20 * time.Millisecond)
}
// Close the snapshoter
close(stopCh)
snap.Wait()
// Make a new client back at the origin.
newClient, err := coordinate.NewClient(coordinate.DefaultConfig())
if err != nil {
t.Fatalf("err: %v", err)
}
// Open the snapshoter
stopCh = make(chan struct{})
_, snap, err = NewSnapshotter(td+"snap", snapshotSizeLimit, false,
logger, clock, newClient, nil, stopCh)
if err != nil {
t.Fatalf("err: %v", err)
}
// Check the values
if snap.LastEventClock() != 1023 {
t.Fatalf("bad clock %d", snap.LastEventClock())
}
if snap.LastQueryClock() != 1023 {
t.Fatalf("bad clock %d", snap.LastQueryClock())
}
coord = newClient.GetCoordinate()
if coord.Vec[0] != 123.4 {
t.Fatalf("bad coordinate: %v", coord)
}
close(stopCh)
snap.Wait()
}
// Create creates a new Serf instance, starting all the background tasks
// to maintain cluster membership information.
//
// After calling this function, the configuration should no longer be used
// or modified by the caller.
func Create(conf *Config) (*Serf, error) {
conf.Init()
if conf.ProtocolVersion < ProtocolVersionMin {
return nil, fmt.Errorf("Protocol version '%d' too low. Must be in range: [%d, %d]",
conf.ProtocolVersion, ProtocolVersionMin, ProtocolVersionMax)
} else if conf.ProtocolVersion > ProtocolVersionMax {
return nil, fmt.Errorf("Protocol version '%d' too high. Must be in range: [%d, %d]",
conf.ProtocolVersion, ProtocolVersionMin, ProtocolVersionMax)
}
serf := &Serf{
config: conf,
logger: log.New(conf.LogOutput, "", log.LstdFlags),
members: make(map[string]*memberState),
queryResponse: make(map[LamportTime]*QueryResponse),
shutdownCh: make(chan struct{}),
state: SerfAlive,
}
// Check that the meta data length is okay
if len(serf.encodeTags(conf.Tags)) > memberlist.MetaMaxSize {
return nil, fmt.Errorf("Encoded length of tags exceeds limit of %d bytes", memberlist.MetaMaxSize)
}
// Check if serf member event coalescing is enabled
if conf.CoalescePeriod > 0 && conf.QuiescentPeriod > 0 && conf.EventCh != nil {
c := &memberEventCoalescer{
lastEvents: make(map[string]EventType),
latestEvents: make(map[string]coalesceEvent),
}
conf.EventCh = coalescedEventCh(conf.EventCh, serf.shutdownCh,
conf.CoalescePeriod, conf.QuiescentPeriod, c)
}
// Check if user event coalescing is enabled
if conf.UserCoalescePeriod > 0 && conf.UserQuiescentPeriod > 0 && conf.EventCh != nil {
c := &userEventCoalescer{
events: make(map[string]*latestUserEvents),
}
conf.EventCh = coalescedEventCh(conf.EventCh, serf.shutdownCh,
conf.UserCoalescePeriod, conf.UserQuiescentPeriod, c)
}
// Listen for internal Serf queries. This is setup before the snapshotter, since
// we want to capture the query-time, but the internal listener does not passthrough
// the queries
outCh, err := newSerfQueries(serf, serf.logger, conf.EventCh, serf.shutdownCh)
if err != nil {
return nil, fmt.Errorf("Failed to setup serf query handler: %v", err)
}
conf.EventCh = outCh
// Set up network coordinate client.
if !conf.DisableCoordinates {
serf.coordClient, err = coordinate.NewClient(coordinate.DefaultConfig())
if err != nil {
return nil, fmt.Errorf("Failed to create coordinate client: %v", err)
}
}
// Try access the snapshot
var oldClock, oldEventClock, oldQueryClock LamportTime
var prev []*PreviousNode
if conf.SnapshotPath != "" {
eventCh, snap, err := NewSnapshotter(
conf.SnapshotPath,
snapshotSizeLimit,
conf.RejoinAfterLeave,
serf.logger,
&serf.clock,
serf.coordClient,
conf.EventCh,
serf.shutdownCh)
if err != nil {
return nil, fmt.Errorf("Failed to setup snapshot: %v", err)
}
serf.snapshotter = snap
conf.EventCh = eventCh
prev = snap.AliveNodes()
oldClock = snap.LastClock()
oldEventClock = snap.LastEventClock()
oldQueryClock = snap.LastQueryClock()
serf.eventMinTime = oldEventClock + 1
serf.queryMinTime = oldQueryClock + 1
}
// Set up the coordinate cache. We do this after we read the snapshot to
// make sure we get a good initial value from there, if we got one.
if !conf.DisableCoordinates {
serf.coordCache = make(map[string]*coordinate.Coordinate)
serf.coordCache[conf.NodeName] = serf.coordClient.GetCoordinate()
}
// Setup the various broadcast queues, which we use to send our own
// custom broadcasts along the gossip channel.
serf.broadcasts = &memberlist.TransmitLimitedQueue{
NumNodes: func() int {
return len(serf.members)
},
RetransmitMult: conf.MemberlistConfig.RetransmitMult,
}
serf.eventBroadcasts = &memberlist.TransmitLimitedQueue{
NumNodes: func() int {
return len(serf.members)
},
RetransmitMult: conf.MemberlistConfig.RetransmitMult,
}
serf.queryBroadcasts = &memberlist.TransmitLimitedQueue{
NumNodes: func() int {
return len(serf.members)
},
RetransmitMult: conf.MemberlistConfig.RetransmitMult,
}
// Create the buffer for recent intents
serf.recentJoin = make([]nodeIntent, conf.RecentIntentBuffer)
serf.recentLeave = make([]nodeIntent, conf.RecentIntentBuffer)
// Create a buffer for events and queries
serf.eventBuffer = make([]*userEvents, conf.EventBuffer)
serf.queryBuffer = make([]*queries, conf.QueryBuffer)
// Ensure our lamport clock is at least 1, so that the default
// join LTime of 0 does not cause issues
serf.clock.Increment()
serf.eventClock.Increment()
serf.queryClock.Increment()
// Restore the clock from snap if we have one
serf.clock.Witness(oldClock)
serf.eventClock.Witness(oldEventClock)
serf.queryClock.Witness(oldQueryClock)
// Modify the memberlist configuration with keys that we set
conf.MemberlistConfig.Events = &eventDelegate{serf: serf}
conf.MemberlistConfig.Conflict = &conflictDelegate{serf: serf}
conf.MemberlistConfig.Delegate = &delegate{serf: serf}
conf.MemberlistConfig.DelegateProtocolVersion = conf.ProtocolVersion
conf.MemberlistConfig.DelegateProtocolMin = ProtocolVersionMin
conf.MemberlistConfig.DelegateProtocolMax = ProtocolVersionMax
conf.MemberlistConfig.Name = conf.NodeName
conf.MemberlistConfig.ProtocolVersion = ProtocolVersionMap[conf.ProtocolVersion]
if !conf.DisableCoordinates {
conf.MemberlistConfig.Ping = &pingDelegate{serf: serf}
}
// Setup a merge delegate if necessary
if conf.Merge != nil {
md := &mergeDelegate{serf: serf}
conf.MemberlistConfig.Merge = md
conf.MemberlistConfig.Alive = md
}
// Create the underlying memberlist that will manage membership
// and failure detection for the Serf instance.
memberlist, err := memberlist.Create(conf.MemberlistConfig)
if err != nil {
return nil, fmt.Errorf("Failed to create memberlist: %v", err)
}
serf.memberlist = memberlist
// Create a key manager for handling all encryption key changes
serf.keyManager = &KeyManager{serf: serf}
// Start the background tasks. See the documentation above each method
// for more information on their role.
go serf.handleReap()
go serf.handleReconnect()
go serf.checkQueueDepth("Intent", serf.broadcasts)
go serf.checkQueueDepth("Event", serf.eventBroadcasts)
go serf.checkQueueDepth("Query", serf.queryBroadcasts)
// Attempt to re-join the cluster if we have known nodes
if len(prev) != 0 {
go serf.handleRejoin(prev)
}
return serf, nil
}
func TestSnapshotter(t *testing.T) {
td, err := ioutil.TempDir("", "serf")
if err != nil {
t.Fatalf("err: %v", err)
}
defer os.RemoveAll(td)
// Set up a coordinate at a known location.
coordClient, err := coordinate.NewClient(coordinate.DefaultConfig())
if err != nil {
t.Fatalf("err: %v", err)
}
coord := coordinate.NewCoordinate(coordinate.DefaultConfig())
coord.Vec[0] = 123.4
coordClient.SetCoordinate(coord)
clock := new(LamportClock)
outCh := make(chan Event, 64)
stopCh := make(chan struct{})
logger := log.New(os.Stderr, "", log.LstdFlags)
inCh, snap, err := NewSnapshotter(td+"snap", snapshotSizeLimit, false,
logger, clock, coordClient, outCh, stopCh)
if err != nil {
t.Fatalf("err: %v", err)
}
// Write some user events
ue := UserEvent{
LTime: 42,
Name: "bar",
}
inCh <- ue
// Write some queries
q := &Query{
LTime: 50,
Name: "uptime",
}
inCh <- q
// Write some member events
clock.Witness(100)
meJoin := MemberEvent{
Type: EventMemberJoin,
Members: []Member{
Member{
Name: "foo",
Addr: []byte{127, 0, 0, 1},
Port: 5000,
},
},
}
meFail := MemberEvent{
Type: EventMemberFailed,
Members: []Member{
Member{
Name: "foo",
Addr: []byte{127, 0, 0, 1},
Port: 5000,
},
},
}
inCh <- meJoin
inCh <- meFail
inCh <- meJoin
// Check these get passed through
select {
case e := <-outCh:
if !reflect.DeepEqual(e, ue) {
t.Fatalf("expected user event: %#v", e)
}
case <-time.After(200 * time.Millisecond):
t.Fatalf("timeout")
}
select {
case e := <-outCh:
if !reflect.DeepEqual(e, q) {
t.Fatalf("expected query event: %#v", e)
}
case <-time.After(200 * time.Millisecond):
t.Fatalf("timeout")
}
select {
case e := <-outCh:
if !reflect.DeepEqual(e, meJoin) {
t.Fatalf("expected member event: %#v", e)
}
case <-time.After(200 * time.Millisecond):
t.Fatalf("timeout")
}
select {
case e := <-outCh:
if !reflect.DeepEqual(e, meFail) {
t.Fatalf("expected member event: %#v", e)
}
case <-time.After(200 * time.Millisecond):
t.Fatalf("timeout")
}
select {
case e := <-outCh:
if !reflect.DeepEqual(e, meJoin) {
t.Fatalf("expected member event: %#v", e)
}
case <-time.After(200 * time.Millisecond):
t.Fatalf("timeout")
}
// Manually kick a coordinate update so the test doesn't have to wait
// for the long period.
snap.updateCoordinate()
// Close the snapshoter
close(stopCh)
snap.Wait()
// Make a new client back at the origin.
newClient, err := coordinate.NewClient(coordinate.DefaultConfig())
if err != nil {
t.Fatalf("err: %v", err)
}
// Open the snapshoter
stopCh = make(chan struct{})
_, snap, err = NewSnapshotter(td+"snap", snapshotSizeLimit, false,
logger, clock, newClient, outCh, stopCh)
if err != nil {
t.Fatalf("err: %v", err)
}
// Check the values
if snap.LastClock() != 100 {
t.Fatalf("bad clock %d", snap.LastClock())
}
if snap.LastEventClock() != 42 {
t.Fatalf("bad clock %d", snap.LastEventClock())
}
if snap.LastQueryClock() != 50 {
t.Fatalf("bad clock %d", snap.LastQueryClock())
}
prev := snap.AliveNodes()
if len(prev) != 1 {
t.Fatalf("expected alive: %#v", prev)
}
if prev[0].Name != "foo" {
t.Fatalf("bad name: %#v", prev[0])
}
if prev[0].Addr != "127.0.0.1:5000" {
t.Fatalf("bad addr: %#v", prev[0])
}
coord = newClient.GetCoordinate()
if coord.Vec[0] != 123.4 {
t.Fatalf("bad coordinate: %v", coord)
}
// Close the snapshotter.
close(stopCh)
snap.Wait()
// Open the snapshotter, make sure nothing dies reading with coordinates
// disabled.
stopCh = make(chan struct{})
_, snap, err = NewSnapshotter(td+"snap", snapshotSizeLimit, false,
logger, clock, nil, outCh, stopCh)
if err != nil {
t.Fatalf("err: %v", err)
}
close(stopCh)
snap.Wait()
}