func TestTribePluginAgreement(t *testing.T) {
numOfTribes := 5
// tribePort := 52600
tribes := getTribes(numOfTribes, nil)
Convey(fmt.Sprintf("%d tribes are started", numOfTribes), t, func() {
for i := 0; i < numOfTribes; i++ {
So(
len(tribes[0].memberlist.Members()),
ShouldEqual,
len(tribes[i].memberlist.Members()),
)
logger.Debugf("%v has %v members", tribes[i].memberlist.LocalNode().Name, len(tribes[i].members))
So(len(tribes[i].members), ShouldEqual, numOfTribes)
}
Convey("The cluster agrees on membership", func() {
for i := 0; i < numOfTribes; i++ {
log.Debugf("%v is reporting %v members", i, len(tribes[i].memberlist.Members()))
So(len(tribes[0].memberlist.Members()), ShouldEqual, len(tribes[i].memberlist.Members()))
So(len(tribes[0].members), ShouldEqual, len(tribes[i].members))
}
oldMember := tribes[0]
err := tribes[0].memberlist.Leave(2 * time.Second)
// err := tribes[0].memberlist.Shutdown()
So(err, ShouldBeNil)
tribes = append(tribes[:0], tribes[1:]...)
Convey("Membership decreases as members leave", func(c C) {
wg := sync.WaitGroup{}
for i := range tribes {
wg.Add(1)
go func(i int) {
defer wg.Done()
for {
if len(tribes[i].members) == len(tribes) {
c.So(len(tribes[i].members), ShouldEqual, len(tribes))
return
}
time.Sleep(20 * time.Millisecond)
}
}(i)
}
wg.Wait()
err := oldMember.memberlist.Shutdown()
So(err, ShouldBeNil)
So(len(tribes[rand.Intn(len(tribes))].memberlist.Members()), ShouldEqual, len(tribes))
So(len(tribes[1].members), ShouldEqual, len(tribes))
Convey("Membership increases as members join", func(c C) {
seed := fmt.Sprintf("%v:%v", tribes[0].memberlist.LocalNode().Addr, tribes[0].memberlist.LocalNode().Port)
conf := getTestConfig()
conf.Name = fmt.Sprintf("member-%d", numOfTribes+1)
conf.Seed = seed
tr, err := New(conf)
if err != nil {
So(err, ShouldBeNil)
}
tribes = append(tribes, tr)
wg := sync.WaitGroup{}
for i := range tribes {
wg.Add(1)
go func(i int) {
defer wg.Done()
for {
if len(tribes[i].memberlist.Members()) == len(tribes) {
c.So(len(tribes[i].members), ShouldEqual, len(tribes))
return
}
time.Sleep(20 * time.Millisecond)
}
}(i)
}
wg.Wait()
So(len(tribes[rand.Intn(len(tribes))].memberlist.Members()), ShouldEqual, len(tribes))
So(len(tribes[rand.Intn(len(tribes))].members), ShouldEqual, len(tribes))
Convey("Handles a 'add agreement' message broadcasted across the cluster", func(c C) {
tribes[0].AddAgreement("clan1")
var wg sync.WaitGroup
for _, t := range tribes {
wg.Add(1)
go func(t *tribe) {
defer wg.Done()
for {
if t.agreements != nil {
if _, ok := t.agreements["clan1"]; ok {
c.So(ok, ShouldEqual, true)
return
}
}
time.Sleep(50 * time.Millisecond)
}
}(t)
}
wg.Wait()
numAddMessages := 10
Convey(fmt.Sprintf("Handles %d plugin 'add messages' broadcasted across the cluster", numAddMessages), func() {
for i := 0; i < numAddMessages; i++ {
tribes[0].AddPlugin("clan1", agreement.Plugin{Name_: fmt.Sprintf("plugin%v", i), Version_: 1})
// time.Sleep(time.Millisecond * 50)
}
wg := sync.WaitGroup{}
for _, tr := range tribes {
wg.Add(1)
go func(tr *tribe) {
defer wg.Done()
for {
if clan, ok := tr.agreements["clan1"]; ok {
if len(clan.PluginAgreement.Plugins) == numAddMessages {
return
}
time.Sleep(50 * time.Millisecond)
log.Debugf("%v has %v of %v plugins and %d intents\n", tr.memberlist.LocalNode().Name, len(clan.PluginAgreement.Plugins), numAddMessages, len(tr.intentBuffer))
}
}
}(tr)
}
log.Debugf("Waits for %d members of clan1 to have %d plugins\n", numOfTribes, numAddMessages)
wg.Wait()
for i := 0; i < numOfTribes; i++ {
So(len(tribes[i].agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, numAddMessages)
logger.Debugf("%v has %v intents\n", tribes[i].memberlist.LocalNode().Name, len(tribes[i].intentBuffer))
So(len(tribes[i].intentBuffer), ShouldEqual, 0)
for k, v := range tribes[i].intentBuffer {
logger.Debugf("\tadd intent %v %v\n", k, v)
}
}
Convey("Handles duplicate 'add plugin' messages", func() {
t := tribes[rand.Intn(numOfTribes)]
msg := &pluginMsg{
Plugin: agreement.Plugin{
Name_: "pluginABC",
Version_: 1,
},
UUID: uuid.New(),
AgreementName: "clan1",
LTime: t.clock.Time(),
Type: addPluginMsgType,
}
So(len(t.intentBuffer), ShouldEqual, 0)
t.handleAddPlugin(msg)
before := len(t.agreements["clan1"].PluginAgreement.Plugins)
t.handleAddPlugin(msg)
after := len(t.agreements["clan1"].PluginAgreement.Plugins)
So(before, ShouldEqual, after)
Convey("Handles out-of-order 'add plugin' messages", func() {
msg := &pluginMsg{
Plugin: agreement.Plugin{
Name_: "pluginABC",
Version_: 1,
},
UUID: uuid.New(),
AgreementName: "clan1",
LTime: t.clock.Time(),
Type: addPluginMsgType,
}
t.handleAddPlugin(msg)
So(len(t.intentBuffer), ShouldEqual, 1)
Convey("Handles duplicate out-of-order 'add plugin' messages", func() {
before := len(t.agreements["clan1"].PluginAgreement.Plugins)
t.handleAddPlugin(msg)
after := len(t.agreements["clan1"].PluginAgreement.Plugins)
So(before, ShouldEqual, after)
So(len(t.intentBuffer), ShouldEqual, 1)
So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldBeGreaterThan, numAddMessages)
t.handleRemovePlugin(&pluginMsg{
LTime: t.clock.Time(),
Plugin: agreement.Plugin{Name_: "pluginABC", Version_: 1},
AgreementName: "clan1",
Type: removePluginMsgType,
})
So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldBeGreaterThan, numAddMessages)
So(len(t.intentBuffer), ShouldEqual, 0)
// removes the plugin added to test duplicates
t.handleRemovePlugin(&pluginMsg{
LTime: t.clock.Time(),
Plugin: agreement.Plugin{Name_: "pluginABC", Version_: 1},
AgreementName: "clan1",
Type: removePluginMsgType,
})
So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, numAddMessages)
// wait for all members of the tribe to get back to 10 plugins
wg = sync.WaitGroup{}
for _, tr := range tribes {
wg.Add(1)
go func(tr *tribe) {
defer wg.Done()
for {
select {
case <-time.After(1500 * time.Millisecond):
c.So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, numAddMessages)
default:
if clan, ok := tr.agreements["clan1"]; ok {
if len(clan.PluginAgreement.Plugins) == numAddMessages {
return
}
time.Sleep(50 * time.Millisecond)
}
}
}
}(tr)
}
Convey("Handles a 'remove plugin' messages broadcasted across the cluster", func(c C) {
for _, t := range tribes {
So(len(t.intentBuffer), ShouldEqual, 0)
So(len(t.intentBuffer), ShouldEqual, 0)
So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, numAddMessages)
}
t := tribes[rand.Intn(numOfTribes)]
plugin := t.agreements["clan1"].PluginAgreement.Plugins[rand.Intn(numAddMessages)]
before := len(t.agreements["clan1"].PluginAgreement.Plugins)
t.RemovePlugin("clan1", plugin)
after := len(t.agreements["clan1"].PluginAgreement.Plugins)
So(before-after, ShouldEqual, 1)
var wg sync.WaitGroup
for _, t := range tribes {
wg.Add(1)
go func(t *tribe) {
defer wg.Done()
for {
select {
case <-time.After(1500 * time.Millisecond):
c.So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, after)
default:
if len(t.agreements["clan1"].PluginAgreement.Plugins) == after {
c.So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, after)
return
}
time.Sleep(50 * time.Millisecond)
}
}
}(t)
}
wg.Done()
Convey("Handles out-of-order remove", func() {
t := tribes[rand.Intn(numOfTribes)]
plugin := t.agreements["clan1"].PluginAgreement.Plugins[rand.Intn(numAddMessages-1)]
msg := &pluginMsg{
LTime: t.clock.Increment(),
Plugin: plugin,
AgreementName: "clan1",
UUID: uuid.New(),
Type: removePluginMsgType,
}
before := len(t.agreements["clan1"].PluginAgreement.Plugins)
t.handleRemovePlugin(msg)
So(before-1, ShouldEqual, len(t.agreements["clan1"].PluginAgreement.Plugins))
before = len(t.agreements["clan1"].PluginAgreement.Plugins)
msg.UUID = uuid.New()
msg.LTime = t.clock.Increment()
t.handleRemovePlugin(msg)
after := len(t.agreements["clan1"].PluginAgreement.Plugins)
So(before, ShouldEqual, after)
So(len(t.intentBuffer), ShouldEqual, 1)
Convey("Handles duplicate out-of-order 'remove plugin' messages", func() {
t.handleRemovePlugin(msg)
after := len(t.agreements["clan1"].PluginAgreement.Plugins)
So(before, ShouldEqual, after)
So(len(t.intentBuffer), ShouldEqual, 1)
t.handleAddPlugin(&pluginMsg{
LTime: t.clock.Increment(),
Plugin: plugin,
AgreementName: "clan1",
Type: addPluginMsgType,
})
So(len(t.intentBuffer), ShouldEqual, 0)
ok, _ := t.agreements["clan1"].PluginAgreement.Plugins.Contains(msg.Plugin)
So(ok, ShouldBeFalse)
Convey("Handles old 'remove plugin' messages", func() {
t := tribes[rand.Intn(numOfTribes)]
plugin := t.agreements["clan1"].PluginAgreement.Plugins[rand.Intn(len(t.agreements["clan1"].PluginAgreement.Plugins))]
msg := &pluginMsg{
LTime: LTime(1025),
Plugin: plugin,
AgreementName: "clan1",
UUID: uuid.New(),
Type: removePluginMsgType,
}
before := len(t.agreements["clan1"].PluginAgreement.Plugins)
t.handleRemovePlugin(msg)
after := len(t.agreements["clan1"].PluginAgreement.Plugins)
So(before-1, ShouldEqual, after)
msg2 := &pluginMsg{
LTime: LTime(513),
Plugin: plugin,
AgreementName: "clan1",
UUID: uuid.New(),
Type: addPluginMsgType,
}
before = len(t.agreements["clan1"].PluginAgreement.Plugins)
t.handleAddPlugin(msg2)
after = len(t.agreements["clan1"].PluginAgreement.Plugins)
So(before, ShouldEqual, after)
msg3 := &pluginMsg{
LTime: LTime(513),
Plugin: plugin,
AgreementName: "clan1",
UUID: uuid.New(),
Type: removePluginMsgType,
}
before = len(t.agreements["clan1"].PluginAgreement.Plugins)
t.handleRemovePlugin(msg3)
after = len(t.agreements["clan1"].PluginAgreement.Plugins)
So(before, ShouldEqual, after)
Convey("The tribe agrees on plugin agreements", func(c C) {
var wg sync.WaitGroup
for _, t := range tribes {
wg.Add(1)
go func(t *tribe) {
for {
defer wg.Done()
select {
case <-time.After(1 * time.Second):
c.So(len(t.memberlist.Members()), ShouldEqual, numOfTribes)
default:
if len(t.agreements["clan1"].PluginAgreement.Plugins) == numAddMessages-1 {
c.So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, numAddMessages-1)
return
}
}
}
}(t)
}
wg.Done()
})
})
})
})
})
})
})
})
})
})
})
})
})
})
}