Example #1
0
func TestTribeFullStateSync(t *testing.T) {
	log.SetLevel(log.WarnLevel)
	tribes := []*tribe{}
	numOfTribes := 5
	agreement1 := "agreement1"
	plugin1 := agreement.Plugin{Name_: "plugin1", Version_: 1, Type_: core.ProcessorPluginType}
	task1 := agreement.Task{ID: uuid.New()}
	Convey("Tribe members are started", t, func() {
		conf := getTestConfig()
		conf.Name = "seed"
		seed, err := New(conf)
		So(seed, ShouldNotBeNil)
		So(err, ShouldBeNil)
		taskManager := &mockTaskManager{}
		seed.SetTaskManager(taskManager)
		tribes = append(tribes, seed)
		for i := 1; i < numOfTribes; i++ {
			conf := getTestConfig()
			conf.Name = fmt.Sprintf("member-%v", i)
			conf.Seed = fmt.Sprintf("%v:%v", "127.0.0.1", seed.memberlist.LocalNode().Port)
			tr, err := New(conf)
			taskManager := &mockTaskManager{}
			tr.SetTaskManager(taskManager)
			So(err, ShouldBeNil)
			So(tr, ShouldNotBeNil)
			tribes = append(tribes, tr)
		}
		var wg sync.WaitGroup
		for _, tr := range tribes {
			timer := time.After(4 * time.Second)
			wg.Add(1)
			go func(tr *tribe) {
				defer wg.Done()
				for {
					select {
					case <-timer:
						panic("timed out establishing membership")
					default:
						if len(tr.members) == len(tribes) {
							return
						}
						logger.Debugf("%v has %v members", tr.memberlist.LocalNode().Name, len(tr.memberlist.Members()))
						time.Sleep(50 * time.Millisecond)
					}
				}
			}(tr)
		}
		wg.Wait()
		Convey("agreements are added", func() {
			t := tribes[rand.Intn(len(tribes))]
			serr := t.AddAgreement(agreement1)
			So(serr, ShouldBeNil)
			err := t.AddPlugin(agreement1, plugin1)
			So(err, ShouldBeNil)
			serr = t.AddTask(agreement1, task1)
			So(serr, ShouldBeNil)
			So(len(t.agreements), ShouldEqual, 1)
			Convey("the state is consistent across the tribe", func() {
				wg = sync.WaitGroup{}
				timedOut := false
				for _, tr := range tribes {
					timer := time.After(10 * time.Second)
					wg.Add(1)
					go func(tr *tribe) {
						defer wg.Done()
						for {
							select {
							case <-timer:
								timedOut = true
								return
							default:
								if a, ok := tr.agreements[agreement1]; ok {
									if a.PluginAgreement != nil {
										if ok, _ := a.PluginAgreement.Plugins.Contains(plugin1); ok {
											return
										}
									}
								}
								logger.Debugf("%v has %v agreements", tr.memberlist.LocalNode().Name, len(tr.agreements))
								time.Sleep(200 * time.Millisecond)
							}
						}
					}(tr)
				}
				wg.Wait()
				So(timedOut, ShouldBeFalse)

				Convey("all members are added to the agreements", func() {
					for _, tr := range tribes {
						logger.Debugf("joining %v %v", agreement1, tr.memberlist.LocalNode().Name)
						err := t.JoinAgreement(agreement1, tr.memberlist.LocalNode().Name)
						So(err, ShouldBeNil)
					}
				})
			})
		})
	})
}
Example #2
0
func TestTribeAgreements(t *testing.T) {
	numOfTribes := 5
	tribes := getTribes(numOfTribes, nil)
	Convey(fmt.Sprintf("%d tribes are started", numOfTribes), t, func() {
		for i := 0; i < numOfTribes; i++ {
			log.Debugf("%v is reporting %v members", i, len(tribes[i].memberlist.Members()))
			So(len(tribes[0].memberlist.Members()), ShouldEqual, len(tribes[i].memberlist.Members()))
		}
		Convey("The cluster agrees on membership", func() {
			for i := 0; i < numOfTribes; i++ {

				So(
					len(tribes[0].memberlist.Members()),
					ShouldEqual,
					len(tribes[i].memberlist.Members()),
				)
				So(len(tribes[0].members), ShouldEqual, len(tribes[i].members))
			}

			Convey("A member handles", func() {
				agreementName := "agreement1"
				t := tribes[0]
				t2 := tribes[1]
				Convey("an out-of-order join agreement message", func() {
					msg := &agreementMsg{
						LTime:         t.clock.Increment(),
						UUID:          uuid.New(),
						AgreementName: agreementName,
						MemberName:    t.memberlist.LocalNode().Name,
						Type:          joinAgreementMsgType,
					}
					msg2 := &agreementMsg{
						LTime:         t.clock.Increment(),
						UUID:          uuid.New(),
						AgreementName: agreementName,
						MemberName:    t2.memberlist.LocalNode().Name,
						Type:          joinAgreementMsgType,
					}

					b := t.handleJoinAgreement(msg)
					So(b, ShouldEqual, true)
					So(len(t.intentBuffer), ShouldEqual, 1)
					t.broadcast(joinAgreementMsgType, msg, nil)
					timer := time.After(2 * time.Second)
				loop1:
					for {
						select {
						case <-timer:
							So("Timed out", ShouldEqual, "")
						default:
							if len(t2.intentBuffer) > 0 {
								break loop1
							}
						}
					}
					So(len(t2.intentBuffer), ShouldEqual, 1)

					b = t.handleJoinAgreement(msg2)
					So(b, ShouldEqual, true)
					So(len(t.intentBuffer), ShouldEqual, 2)
					t.broadcast(joinAgreementMsgType, msg2, nil)

					timer = time.After(2 * time.Second)
				loop2:
					for {
						select {
						case <-timer:
							So("Timed out", ShouldEqual, "")
						default:
							if len(t2.intentBuffer) == 2 {
								break loop2
							}
						}
					}
					So(len(t2.intentBuffer), ShouldEqual, 2)

					Convey("an out-of-order add plugin message", func() {
						plugin := agreement.Plugin{Name_: "plugin1", Version_: 1}
						msg := &pluginMsg{
							LTime:         t.clock.Increment(),
							UUID:          uuid.New(),
							Plugin:        plugin,
							AgreementName: agreementName,
							Type:          addPluginMsgType,
						}
						b := t.handleAddPlugin(msg)
						So(b, ShouldEqual, true)
						So(len(t.intentBuffer), ShouldEqual, 3)
						t.broadcast(addPluginMsgType, msg, nil)

						Convey("an add agreement", func() {
							err := t.AddAgreement(agreementName)
							So(err, ShouldBeNil)
							err = t.AddAgreement(agreementName)
							So(err.Error(), ShouldResemble, errAgreementAlreadyExists.Error())
							var wg sync.WaitGroup
							for _, t := range tribes {
								wg.Add(1)
								go func(t *tribe) {
									defer wg.Done()
									for {
										if a, ok := t.agreements[agreementName]; ok {
											logger.Debugf("%s has %d plugins in agreement '%s' and %d intents", t.memberlist.LocalNode().Name, len(t.agreements[agreementName].PluginAgreement.Plugins), agreementName, len(t.intentBuffer))
											if ok, _ := a.PluginAgreement.Plugins.Contains(plugin); ok {
												if len(t.intentBuffer) == 0 {
													return
												}
											}
										}
										logger.Debugf("%s has %d intents", t.memberlist.LocalNode().Name, len(t.intentBuffer))
										time.Sleep(50 * time.Millisecond)
									}
								}(t)
							}
							wg.Wait()

							Convey("being added to an agreement it already belongs to", func() {
								err := t.JoinAgreement(agreementName, t.memberlist.LocalNode().Name)
								So(err.Error(), ShouldResemble, errAlreadyMemberOfPluginAgreement.Error())

								Convey("leaving an agreement that doesn't exist", func() {
									err := t.LeaveAgreement("whatever", t.memberlist.LocalNode().Name)
									So(err.Error(), ShouldResemble, errAgreementDoesNotExist.Error())

									Convey("an unknown member trying to leave an agreement", func() {
										err := t.LeaveAgreement(agreementName, "whatever")
										So(err.Error(), ShouldResemble, errUnknownMember.Error())

										Convey("a member leaving an agreement it isn't part of", func() {
											err := t.LeaveAgreement(agreementName, tribes[2].memberlist.LocalNode().Name)
											So(err, ShouldNotBeNil)
											So(err.Error(), ShouldResemble, errNotAMember.Error())

											Convey("an unknown member trying to join an agreement", func() {
												msg := &agreementMsg{
													LTime:         t.clock.Time(),
													UUID:          uuid.New(),
													AgreementName: agreementName,
													MemberName:    "whatever",
													Type:          joinAgreementMsgType,
												}
												err := t.joinAgreement(msg)
												So(err, ShouldNotBeNil)
												So(err.Error(), ShouldResemble, errUnknownMember.Error())

												Convey("leaving an agreement", func() {
													So(len(t.agreements[agreementName].Members), ShouldEqual, 2)
													So(t.members[t.memberlist.LocalNode().Name].PluginAgreement, ShouldNotBeNil)
													err := t.LeaveAgreement(agreementName, t.memberlist.LocalNode().Name)
													So(err, ShouldBeNil)
													So(len(t.agreements[agreementName].Members), ShouldEqual, 1)
													So(t.members[t.memberlist.LocalNode().Name].PluginAgreement, ShouldBeNil)
													Convey("leaving a tribe results in the member leaving the agreement", func() {
														t2.memberlist.Leave(500 * time.Millisecond)
														timer := time.After(2 * time.Second)
													loop3:
														for {
															select {
															case <-timer:
																So("Timed out", ShouldEqual, "")
															default:
																if len(t.agreements[agreementName].Members) == 0 {
																	break loop3
																}
															}
														}
														So(len(t.agreements[agreementName].Members), ShouldEqual, 0)
														Convey("removes an agreement", func() {
															err := t.RemoveAgreement(agreementName)
															So(err, ShouldBeNil)
															So(len(t.agreements), ShouldEqual, 0)
															Convey("removes an agreement that no longer exists", func() {
																err := t.RemoveAgreement(agreementName)
																So(err.Error(), ShouldResemble, errAgreementDoesNotExist.Error())
																So(len(t.agreements), ShouldEqual, 0)
															})
														})
													})
												})
											})
										})
									})
								})
							})
						})
					})
				})
			})
		})
	})
}
Example #3
0
func TestTribeMembership(t *testing.T) {
	numOfTribes := 5
	tribes := getTribes(numOfTribes, nil)
	Convey(fmt.Sprintf("%d tribes are started", numOfTribes), t, func() {
		for i := 0; i < numOfTribes; i++ {
			log.Debugf("%v is reporting %v members", i, len(tribes[i].memberlist.Members()))
			So(len(tribes[0].memberlist.Members()), ShouldEqual, len(tribes[i].memberlist.Members()))
		}
		Convey("The cluster agrees on membership", func() {
			for i := 0; i < numOfTribes; i++ {

				So(
					len(tribes[0].memberlist.Members()),
					ShouldEqual,
					len(tribes[i].memberlist.Members()),
				)
				So(len(tribes[0].members), ShouldEqual, len(tribes[i].members))
			}
			Convey("Adds an agreement", func(c C) {
				a := "agreement1"
				t := tribes[numOfTribes-1]
				t.AddAgreement("agreement1")
				var wg sync.WaitGroup
				for _, t := range tribes {
					wg.Add(1)
					go func(t *tribe) {
						defer wg.Done()
						for {
							if t.agreements != nil {
								if _, ok := t.agreements[a]; ok {
									c.So(ok, ShouldEqual, true)
									return
								}
								logger.Debugf(
									"%v has %d agreements",
									t.memberlist.LocalNode().Name,
									len(t.agreements),
								)
							}
							time.Sleep(50 * time.Millisecond)
						}
					}(t)
				}
				wg.Wait()
				for _, t := range tribes {
					So(len(t.agreements), ShouldEqual, 1)
					So(t.agreements[a], ShouldNotBeNil)
				}
				Convey("A member", func() {
					Convey("joins an agreement", func() {
						err := t.JoinAgreement(a, t.memberlist.LocalNode().Name)
						So(err, ShouldBeNil)
					})
					Convey("is added to an agreement it already belongs to", func() {
						Convey("adds a plugin to agreement", func() {
							err := t.AddPlugin(a, agreement.Plugin{Name_: "plugin1", Version_: 1})
							So(err, ShouldBeNil)
						})

						err := t.JoinAgreement(a, t.memberlist.LocalNode().Name)
						So(err.Error(), ShouldResemble, errAlreadyMemberOfPluginAgreement.Error())
					})
					Convey("leaves an agreement that doesn't exist", func() {
						err := t.LeaveAgreement("whatever", t.memberlist.LocalNode().Name)
						So(err.Error(), ShouldResemble, errAgreementDoesNotExist.Error())
					})
					Convey("handles an unknown member trying to leave an agreement", func() {
						err := t.LeaveAgreement(a, "whatever")
						So(err.Error(), ShouldResemble, errUnknownMember.Error())
					})
					Convey("handles a member leaving an agreement it isn't part of", func() {
						err := t.LeaveAgreement(a, tribes[0].memberlist.LocalNode().Name)
						So(err.Error(), ShouldResemble, errNotAMember.Error())
					})
					Convey("handles an unknown member trying to join an agreement", func() {
						msg := &agreementMsg{
							LTime:         t.clock.Time(),
							UUID:          uuid.New(),
							AgreementName: a,
							MemberName:    "whatever",
							Type:          joinAgreementMsgType,
						}
						err := t.joinAgreement(msg)
						So(err, ShouldNotBeNil)
						So(err.Error(), ShouldResemble, errUnknownMember.Error())
					})
				})
			})
		})

	})
}
Example #4
0
func TestTribeTaskAgreements(t *testing.T) {
	numOfTribes := 5
	tribes := getTribes(numOfTribes, nil)
	Convey(fmt.Sprintf("%d tribes are started", numOfTribes), t, func() {
		for i := 0; i < numOfTribes; i++ {
			log.Debugf("%v is reporting %v members", i, len(tribes[i].memberlist.Members()))
			So(len(tribes[0].memberlist.Members()), ShouldEqual, len(tribes[i].memberlist.Members()))
		}
		Convey("the cluster agrees on membership", func() {
			for i := 0; i < numOfTribes; i++ {
				So(
					len(tribes[0].memberlist.Members()),
					ShouldEqual,
					len(tribes[i].memberlist.Members()),
				)
				So(len(tribes[0].members), ShouldEqual, len(tribes[i].members))
			}

			agreementName := "agreement1"
			agreementName2 := "agreement2"
			task1 := agreement.Task{ID: uuid.New()}
			task2 := agreement.Task{ID: uuid.New()}
			Convey("a member handles", func() {
				t := tribes[0]
				t2 := tribes[1]
				Convey("an out of order 'add task' message", func() {
					msg := &taskMsg{
						LTime:         t.clock.Increment(),
						UUID:          uuid.New(),
						TaskID:        task1.ID,
						AgreementName: agreementName,
						Type:          addTaskMsgType,
					}
					b := t.handleAddTask(msg)
					So(b, ShouldEqual, true)
					t.broadcast(addTaskMsgType, msg, nil)
					So(len(t.intentBuffer), ShouldEqual, 1)
					err := t.AddTask(agreementName, task1)
					So(err.Error(), ShouldResemble, errAgreementDoesNotExist.Error())
					err = t.AddAgreement(agreementName)
					So(err, ShouldBeNil)
					So(len(t.intentBuffer), ShouldEqual, 0)
					So(len(t.agreements[agreementName].TaskAgreement.Tasks), ShouldEqual, 1)
					ok, _ := t.agreements[agreementName].TaskAgreement.Tasks.Contains(task1)
					So(ok, ShouldBeTrue)
					Convey("adding an existing task", func() {
						err := t.AddTask(agreementName, task1)
						So(err.Error(), ShouldResemble, errTaskAlreadyExists.Error())
						Convey("removing a task that doesn't exist", func() {
							err := t.RemoveTask(agreementName, agreement.Task{ID: uuid.New()})
							So(err.Error(), ShouldResemble, errTaskDoesNotExist.Error())
							err = t.RemoveTask("doesn't exist", task1)
							So(err.Error(), ShouldResemble, errAgreementDoesNotExist.Error())
							Convey("joining an agreement with tasks", func() {
								err := t.AddAgreement(agreementName2)
								So(err, ShouldBeNil)
								err = t.AddTask(agreementName2, task2)
								So(err, ShouldBeNil)
								err = t.JoinAgreement(agreementName, t.memberlist.LocalNode().Name)
								So(err, ShouldBeNil)
								err = t.JoinAgreement(agreementName, t2.memberlist.LocalNode().Name)
								So(err, ShouldBeNil)
								err = t.JoinAgreement(agreementName2, t.memberlist.LocalNode().Name)
								So(err, ShouldBeNil)
								So(len(t.members[t.memberlist.LocalNode().Name].TaskAgreements), ShouldEqual, 2)
								err = t.canJoinAgreement(agreementName2, t.memberlist.LocalNode().Name)
								So(err, ShouldBeNil)
								So(t.members[t.memberlist.LocalNode().Name].PluginAgreement, ShouldNotBeNil)
								So(len(t.members[t.memberlist.LocalNode().Name].TaskAgreements), ShouldEqual, 2)
								Convey("all members agree on tasks", func(c C) {
									var wg sync.WaitGroup
									for _, t := range tribes {
										wg.Add(1)
										go func(t *tribe) {
											defer wg.Done()
											for {
												if a, ok := t.agreements[agreementName]; ok {
													if ok, _ := a.TaskAgreement.Tasks.Contains(task1); ok {
														return
													}
												}
												time.Sleep(50 * time.Millisecond)
											}
										}(t)
									}
									wg.Wait()
									Convey("the agreement is queried for the state of a given task", func() {
										t := tribes[rand.Intn(numOfTribes)]
										resp := t.taskStateQuery(agreementName, task1.ID)
										So(resp, ShouldNotBeNil)
										responses := taskStateResponses{}
										for r := range resp.resp {
											responses = append(responses, r)
										}
										So(len(responses), ShouldEqual, 2)
										So(responses.State(), ShouldEqual, core.TaskSpinning)
										Convey("a member handles removing a task", func() {
											t := tribes[rand.Intn(numOfTribes)]
											ok, _ := t.agreements[agreementName].TaskAgreement.Tasks.Contains(task1)
											So(ok, ShouldBeTrue)
											err := t.RemoveTask(agreementName, task1)
											ok, _ = t.agreements[agreementName].TaskAgreement.Tasks.Contains(task1)
											So(ok, ShouldBeFalse)
											So(err, ShouldBeNil)
											So(t.intentBuffer, ShouldBeEmpty)
											var wg sync.WaitGroup
											for _, t := range tribes {
												wg.Add(1)
												go func(t *tribe) {
													defer wg.Done()
													for {
														if a, ok := t.agreements[agreementName]; ok {
															if len(a.TaskAgreement.Tasks) == 0 {
																return
															}
														}
														time.Sleep(50 * time.Millisecond)
													}
												}(t)
											}
											wg.Wait()
										})
									})
								})
							})
						})
					})
				})
			})
		})
	})
}