func TestTribeFullStateSync(t *testing.T) {
log.SetLevel(log.WarnLevel)
tribes := []*tribe{}
numOfTribes := 5
agreement1 := "agreement1"
plugin1 := agreement.Plugin{Name_: "plugin1", Version_: 1, Type_: core.ProcessorPluginType}
task1 := agreement.Task{ID: uuid.New()}
Convey("Tribe members are started", t, func() {
conf := getTestConfig()
conf.Name = "seed"
seed, err := New(conf)
So(seed, ShouldNotBeNil)
So(err, ShouldBeNil)
taskManager := &mockTaskManager{}
seed.SetTaskManager(taskManager)
tribes = append(tribes, seed)
for i := 1; i < numOfTribes; i++ {
conf := getTestConfig()
conf.Name = fmt.Sprintf("member-%v", i)
conf.Seed = fmt.Sprintf("%v:%v", "127.0.0.1", seed.memberlist.LocalNode().Port)
tr, err := New(conf)
taskManager := &mockTaskManager{}
tr.SetTaskManager(taskManager)
So(err, ShouldBeNil)
So(tr, ShouldNotBeNil)
tribes = append(tribes, tr)
}
var wg sync.WaitGroup
for _, tr := range tribes {
timer := time.After(4 * time.Second)
wg.Add(1)
go func(tr *tribe) {
defer wg.Done()
for {
select {
case <-timer:
panic("timed out establishing membership")
default:
if len(tr.members) == len(tribes) {
return
}
logger.Debugf("%v has %v members", tr.memberlist.LocalNode().Name, len(tr.memberlist.Members()))
time.Sleep(50 * time.Millisecond)
}
}
}(tr)
}
wg.Wait()
Convey("agreements are added", func() {
t := tribes[rand.Intn(len(tribes))]
serr := t.AddAgreement(agreement1)
So(serr, ShouldBeNil)
err := t.AddPlugin(agreement1, plugin1)
So(err, ShouldBeNil)
serr = t.AddTask(agreement1, task1)
So(serr, ShouldBeNil)
So(len(t.agreements), ShouldEqual, 1)
Convey("the state is consistent across the tribe", func() {
wg = sync.WaitGroup{}
timedOut := false
for _, tr := range tribes {
timer := time.After(10 * time.Second)
wg.Add(1)
go func(tr *tribe) {
defer wg.Done()
for {
select {
case <-timer:
timedOut = true
return
default:
if a, ok := tr.agreements[agreement1]; ok {
if a.PluginAgreement != nil {
if ok, _ := a.PluginAgreement.Plugins.Contains(plugin1); ok {
return
}
}
}
logger.Debugf("%v has %v agreements", tr.memberlist.LocalNode().Name, len(tr.agreements))
time.Sleep(200 * time.Millisecond)
}
}
}(tr)
}
wg.Wait()
So(timedOut, ShouldBeFalse)
Convey("all members are added to the agreements", func() {
for _, tr := range tribes {
logger.Debugf("joining %v %v", agreement1, tr.memberlist.LocalNode().Name)
err := t.JoinAgreement(agreement1, tr.memberlist.LocalNode().Name)
So(err, ShouldBeNil)
}
})
})
})
})
}
func TestTribeAgreements(t *testing.T) {
numOfTribes := 5
tribes := getTribes(numOfTribes, nil)
Convey(fmt.Sprintf("%d tribes are started", numOfTribes), t, func() {
for i := 0; i < numOfTribes; i++ {
log.Debugf("%v is reporting %v members", i, len(tribes[i].memberlist.Members()))
So(len(tribes[0].memberlist.Members()), ShouldEqual, len(tribes[i].memberlist.Members()))
}
Convey("The cluster agrees on membership", func() {
for i := 0; i < numOfTribes; i++ {
So(
len(tribes[0].memberlist.Members()),
ShouldEqual,
len(tribes[i].memberlist.Members()),
)
So(len(tribes[0].members), ShouldEqual, len(tribes[i].members))
}
Convey("A member handles", func() {
agreementName := "agreement1"
t := tribes[0]
t2 := tribes[1]
Convey("an out-of-order join agreement message", func() {
msg := &agreementMsg{
LTime: t.clock.Increment(),
UUID: uuid.New(),
AgreementName: agreementName,
MemberName: t.memberlist.LocalNode().Name,
Type: joinAgreementMsgType,
}
msg2 := &agreementMsg{
LTime: t.clock.Increment(),
UUID: uuid.New(),
AgreementName: agreementName,
MemberName: t2.memberlist.LocalNode().Name,
Type: joinAgreementMsgType,
}
b := t.handleJoinAgreement(msg)
So(b, ShouldEqual, true)
So(len(t.intentBuffer), ShouldEqual, 1)
t.broadcast(joinAgreementMsgType, msg, nil)
timer := time.After(2 * time.Second)
loop1:
for {
select {
case <-timer:
So("Timed out", ShouldEqual, "")
default:
if len(t2.intentBuffer) > 0 {
break loop1
}
}
}
So(len(t2.intentBuffer), ShouldEqual, 1)
b = t.handleJoinAgreement(msg2)
So(b, ShouldEqual, true)
So(len(t.intentBuffer), ShouldEqual, 2)
t.broadcast(joinAgreementMsgType, msg2, nil)
timer = time.After(2 * time.Second)
loop2:
for {
select {
case <-timer:
So("Timed out", ShouldEqual, "")
default:
if len(t2.intentBuffer) == 2 {
break loop2
}
}
}
So(len(t2.intentBuffer), ShouldEqual, 2)
Convey("an out-of-order add plugin message", func() {
plugin := agreement.Plugin{Name_: "plugin1", Version_: 1}
msg := &pluginMsg{
LTime: t.clock.Increment(),
UUID: uuid.New(),
Plugin: plugin,
AgreementName: agreementName,
Type: addPluginMsgType,
}
b := t.handleAddPlugin(msg)
So(b, ShouldEqual, true)
So(len(t.intentBuffer), ShouldEqual, 3)
t.broadcast(addPluginMsgType, msg, nil)
Convey("an add agreement", func() {
err := t.AddAgreement(agreementName)
So(err, ShouldBeNil)
err = t.AddAgreement(agreementName)
So(err.Error(), ShouldResemble, errAgreementAlreadyExists.Error())
var wg sync.WaitGroup
for _, t := range tribes {
wg.Add(1)
go func(t *tribe) {
defer wg.Done()
for {
if a, ok := t.agreements[agreementName]; ok {
logger.Debugf("%s has %d plugins in agreement '%s' and %d intents", t.memberlist.LocalNode().Name, len(t.agreements[agreementName].PluginAgreement.Plugins), agreementName, len(t.intentBuffer))
if ok, _ := a.PluginAgreement.Plugins.Contains(plugin); ok {
if len(t.intentBuffer) == 0 {
return
}
}
}
logger.Debugf("%s has %d intents", t.memberlist.LocalNode().Name, len(t.intentBuffer))
time.Sleep(50 * time.Millisecond)
}
}(t)
}
wg.Wait()
Convey("being added to an agreement it already belongs to", func() {
err := t.JoinAgreement(agreementName, t.memberlist.LocalNode().Name)
So(err.Error(), ShouldResemble, errAlreadyMemberOfPluginAgreement.Error())
Convey("leaving an agreement that doesn't exist", func() {
err := t.LeaveAgreement("whatever", t.memberlist.LocalNode().Name)
So(err.Error(), ShouldResemble, errAgreementDoesNotExist.Error())
Convey("an unknown member trying to leave an agreement", func() {
err := t.LeaveAgreement(agreementName, "whatever")
So(err.Error(), ShouldResemble, errUnknownMember.Error())
Convey("a member leaving an agreement it isn't part of", func() {
err := t.LeaveAgreement(agreementName, tribes[2].memberlist.LocalNode().Name)
So(err, ShouldNotBeNil)
So(err.Error(), ShouldResemble, errNotAMember.Error())
Convey("an unknown member trying to join an agreement", func() {
msg := &agreementMsg{
LTime: t.clock.Time(),
UUID: uuid.New(),
AgreementName: agreementName,
MemberName: "whatever",
Type: joinAgreementMsgType,
}
err := t.joinAgreement(msg)
So(err, ShouldNotBeNil)
So(err.Error(), ShouldResemble, errUnknownMember.Error())
Convey("leaving an agreement", func() {
So(len(t.agreements[agreementName].Members), ShouldEqual, 2)
So(t.members[t.memberlist.LocalNode().Name].PluginAgreement, ShouldNotBeNil)
err := t.LeaveAgreement(agreementName, t.memberlist.LocalNode().Name)
So(err, ShouldBeNil)
So(len(t.agreements[agreementName].Members), ShouldEqual, 1)
So(t.members[t.memberlist.LocalNode().Name].PluginAgreement, ShouldBeNil)
Convey("leaving a tribe results in the member leaving the agreement", func() {
t2.memberlist.Leave(500 * time.Millisecond)
timer := time.After(2 * time.Second)
loop3:
for {
select {
case <-timer:
So("Timed out", ShouldEqual, "")
default:
if len(t.agreements[agreementName].Members) == 0 {
break loop3
}
}
}
So(len(t.agreements[agreementName].Members), ShouldEqual, 0)
Convey("removes an agreement", func() {
err := t.RemoveAgreement(agreementName)
So(err, ShouldBeNil)
So(len(t.agreements), ShouldEqual, 0)
Convey("removes an agreement that no longer exists", func() {
err := t.RemoveAgreement(agreementName)
So(err.Error(), ShouldResemble, errAgreementDoesNotExist.Error())
So(len(t.agreements), ShouldEqual, 0)
})
})
})
})
})
})
})
})
})
})
})
})
})
})
})
}
func TestTribeMembership(t *testing.T) {
numOfTribes := 5
tribes := getTribes(numOfTribes, nil)
Convey(fmt.Sprintf("%d tribes are started", numOfTribes), t, func() {
for i := 0; i < numOfTribes; i++ {
log.Debugf("%v is reporting %v members", i, len(tribes[i].memberlist.Members()))
So(len(tribes[0].memberlist.Members()), ShouldEqual, len(tribes[i].memberlist.Members()))
}
Convey("The cluster agrees on membership", func() {
for i := 0; i < numOfTribes; i++ {
So(
len(tribes[0].memberlist.Members()),
ShouldEqual,
len(tribes[i].memberlist.Members()),
)
So(len(tribes[0].members), ShouldEqual, len(tribes[i].members))
}
Convey("Adds an agreement", func(c C) {
a := "agreement1"
t := tribes[numOfTribes-1]
t.AddAgreement("agreement1")
var wg sync.WaitGroup
for _, t := range tribes {
wg.Add(1)
go func(t *tribe) {
defer wg.Done()
for {
if t.agreements != nil {
if _, ok := t.agreements[a]; ok {
c.So(ok, ShouldEqual, true)
return
}
logger.Debugf(
"%v has %d agreements",
t.memberlist.LocalNode().Name,
len(t.agreements),
)
}
time.Sleep(50 * time.Millisecond)
}
}(t)
}
wg.Wait()
for _, t := range tribes {
So(len(t.agreements), ShouldEqual, 1)
So(t.agreements[a], ShouldNotBeNil)
}
Convey("A member", func() {
Convey("joins an agreement", func() {
err := t.JoinAgreement(a, t.memberlist.LocalNode().Name)
So(err, ShouldBeNil)
})
Convey("is added to an agreement it already belongs to", func() {
Convey("adds a plugin to agreement", func() {
err := t.AddPlugin(a, agreement.Plugin{Name_: "plugin1", Version_: 1})
So(err, ShouldBeNil)
})
err := t.JoinAgreement(a, t.memberlist.LocalNode().Name)
So(err.Error(), ShouldResemble, errAlreadyMemberOfPluginAgreement.Error())
})
Convey("leaves an agreement that doesn't exist", func() {
err := t.LeaveAgreement("whatever", t.memberlist.LocalNode().Name)
So(err.Error(), ShouldResemble, errAgreementDoesNotExist.Error())
})
Convey("handles an unknown member trying to leave an agreement", func() {
err := t.LeaveAgreement(a, "whatever")
So(err.Error(), ShouldResemble, errUnknownMember.Error())
})
Convey("handles a member leaving an agreement it isn't part of", func() {
err := t.LeaveAgreement(a, tribes[0].memberlist.LocalNode().Name)
So(err.Error(), ShouldResemble, errNotAMember.Error())
})
Convey("handles an unknown member trying to join an agreement", func() {
msg := &agreementMsg{
LTime: t.clock.Time(),
UUID: uuid.New(),
AgreementName: a,
MemberName: "whatever",
Type: joinAgreementMsgType,
}
err := t.joinAgreement(msg)
So(err, ShouldNotBeNil)
So(err.Error(), ShouldResemble, errUnknownMember.Error())
})
})
})
})
})
}
func TestTribeTaskAgreements(t *testing.T) {
numOfTribes := 5
tribes := getTribes(numOfTribes, nil)
Convey(fmt.Sprintf("%d tribes are started", numOfTribes), t, func() {
for i := 0; i < numOfTribes; i++ {
log.Debugf("%v is reporting %v members", i, len(tribes[i].memberlist.Members()))
So(len(tribes[0].memberlist.Members()), ShouldEqual, len(tribes[i].memberlist.Members()))
}
Convey("the cluster agrees on membership", func() {
for i := 0; i < numOfTribes; i++ {
So(
len(tribes[0].memberlist.Members()),
ShouldEqual,
len(tribes[i].memberlist.Members()),
)
So(len(tribes[0].members), ShouldEqual, len(tribes[i].members))
}
agreementName := "agreement1"
agreementName2 := "agreement2"
task1 := agreement.Task{ID: uuid.New()}
task2 := agreement.Task{ID: uuid.New()}
Convey("a member handles", func() {
t := tribes[0]
t2 := tribes[1]
Convey("an out of order 'add task' message", func() {
msg := &taskMsg{
LTime: t.clock.Increment(),
UUID: uuid.New(),
TaskID: task1.ID,
AgreementName: agreementName,
Type: addTaskMsgType,
}
b := t.handleAddTask(msg)
So(b, ShouldEqual, true)
t.broadcast(addTaskMsgType, msg, nil)
So(len(t.intentBuffer), ShouldEqual, 1)
err := t.AddTask(agreementName, task1)
So(err.Error(), ShouldResemble, errAgreementDoesNotExist.Error())
err = t.AddAgreement(agreementName)
So(err, ShouldBeNil)
So(len(t.intentBuffer), ShouldEqual, 0)
So(len(t.agreements[agreementName].TaskAgreement.Tasks), ShouldEqual, 1)
ok, _ := t.agreements[agreementName].TaskAgreement.Tasks.Contains(task1)
So(ok, ShouldBeTrue)
Convey("adding an existing task", func() {
err := t.AddTask(agreementName, task1)
So(err.Error(), ShouldResemble, errTaskAlreadyExists.Error())
Convey("removing a task that doesn't exist", func() {
err := t.RemoveTask(agreementName, agreement.Task{ID: uuid.New()})
So(err.Error(), ShouldResemble, errTaskDoesNotExist.Error())
err = t.RemoveTask("doesn't exist", task1)
So(err.Error(), ShouldResemble, errAgreementDoesNotExist.Error())
Convey("joining an agreement with tasks", func() {
err := t.AddAgreement(agreementName2)
So(err, ShouldBeNil)
err = t.AddTask(agreementName2, task2)
So(err, ShouldBeNil)
err = t.JoinAgreement(agreementName, t.memberlist.LocalNode().Name)
So(err, ShouldBeNil)
err = t.JoinAgreement(agreementName, t2.memberlist.LocalNode().Name)
So(err, ShouldBeNil)
err = t.JoinAgreement(agreementName2, t.memberlist.LocalNode().Name)
So(err, ShouldBeNil)
So(len(t.members[t.memberlist.LocalNode().Name].TaskAgreements), ShouldEqual, 2)
err = t.canJoinAgreement(agreementName2, t.memberlist.LocalNode().Name)
So(err, ShouldBeNil)
So(t.members[t.memberlist.LocalNode().Name].PluginAgreement, ShouldNotBeNil)
So(len(t.members[t.memberlist.LocalNode().Name].TaskAgreements), ShouldEqual, 2)
Convey("all members agree on tasks", func(c C) {
var wg sync.WaitGroup
for _, t := range tribes {
wg.Add(1)
go func(t *tribe) {
defer wg.Done()
for {
if a, ok := t.agreements[agreementName]; ok {
if ok, _ := a.TaskAgreement.Tasks.Contains(task1); ok {
return
}
}
time.Sleep(50 * time.Millisecond)
}
}(t)
}
wg.Wait()
Convey("the agreement is queried for the state of a given task", func() {
t := tribes[rand.Intn(numOfTribes)]
resp := t.taskStateQuery(agreementName, task1.ID)
So(resp, ShouldNotBeNil)
responses := taskStateResponses{}
for r := range resp.resp {
responses = append(responses, r)
}
So(len(responses), ShouldEqual, 2)
So(responses.State(), ShouldEqual, core.TaskSpinning)
Convey("a member handles removing a task", func() {
t := tribes[rand.Intn(numOfTribes)]
ok, _ := t.agreements[agreementName].TaskAgreement.Tasks.Contains(task1)
So(ok, ShouldBeTrue)
err := t.RemoveTask(agreementName, task1)
ok, _ = t.agreements[agreementName].TaskAgreement.Tasks.Contains(task1)
So(ok, ShouldBeFalse)
So(err, ShouldBeNil)
So(t.intentBuffer, ShouldBeEmpty)
var wg sync.WaitGroup
for _, t := range tribes {
wg.Add(1)
go func(t *tribe) {
defer wg.Done()
for {
if a, ok := t.agreements[agreementName]; ok {
if len(a.TaskAgreement.Tasks) == 0 {
return
}
}
time.Sleep(50 * time.Millisecond)
}
}(t)
}
wg.Wait()
})
})
})
})
})
})
})
})
})
})
}