Esempio n. 1
0
func TestTribePluginAgreement(t *testing.T) {
	numOfTribes := 5
	// tribePort := 52600
	tribes := getTribes(numOfTribes, nil)
	Convey(fmt.Sprintf("%d tribes are started", numOfTribes), t, func() {
		for i := 0; i < numOfTribes; i++ {
			So(
				len(tribes[0].memberlist.Members()),
				ShouldEqual,
				len(tribes[i].memberlist.Members()),
			)
			logger.Debugf("%v has %v members", tribes[i].memberlist.LocalNode().Name, len(tribes[i].members))
			So(len(tribes[i].members), ShouldEqual, numOfTribes)
		}

		Convey("The cluster agrees on membership", func() {
			for i := 0; i < numOfTribes; i++ {
				log.Debugf("%v is reporting %v members", i, len(tribes[i].memberlist.Members()))
				So(len(tribes[0].memberlist.Members()), ShouldEqual, len(tribes[i].memberlist.Members()))
				So(len(tribes[0].members), ShouldEqual, len(tribes[i].members))
			}
			oldMember := tribes[0]
			err := tribes[0].memberlist.Leave(2 * time.Second)
			// err := tribes[0].memberlist.Shutdown()
			So(err, ShouldBeNil)
			tribes = append(tribes[:0], tribes[1:]...)

			Convey("Membership decreases as members leave", func(c C) {
				wg := sync.WaitGroup{}
				for i := range tribes {
					wg.Add(1)
					go func(i int) {
						defer wg.Done()
						for {
							if len(tribes[i].members) == len(tribes) {
								c.So(len(tribes[i].members), ShouldEqual, len(tribes))
								return
							}
							time.Sleep(20 * time.Millisecond)
						}
					}(i)
				}
				wg.Wait()
				err := oldMember.memberlist.Shutdown()
				So(err, ShouldBeNil)
				So(len(tribes[rand.Intn(len(tribes))].memberlist.Members()), ShouldEqual, len(tribes))
				So(len(tribes[1].members), ShouldEqual, len(tribes))

				Convey("Membership increases as members join", func(c C) {
					seed := fmt.Sprintf("%v:%v", tribes[0].memberlist.LocalNode().Addr, tribes[0].memberlist.LocalNode().Port)
					conf := getTestConfig()
					conf.Name = fmt.Sprintf("member-%d", numOfTribes+1)
					conf.Seed = seed
					tr, err := New(conf)
					if err != nil {
						So(err, ShouldBeNil)
					}
					tribes = append(tribes, tr)

					wg := sync.WaitGroup{}
					for i := range tribes {
						wg.Add(1)
						go func(i int) {
							defer wg.Done()
							for {
								if len(tribes[i].memberlist.Members()) == len(tribes) {
									c.So(len(tribes[i].members), ShouldEqual, len(tribes))
									return
								}
								time.Sleep(20 * time.Millisecond)
							}
						}(i)
					}
					wg.Wait()
					So(len(tribes[rand.Intn(len(tribes))].memberlist.Members()), ShouldEqual, len(tribes))
					So(len(tribes[rand.Intn(len(tribes))].members), ShouldEqual, len(tribes))

					Convey("Handles a 'add agreement' message broadcasted across the cluster", func(c C) {
						tribes[0].AddAgreement("clan1")
						var wg sync.WaitGroup
						for _, t := range tribes {
							wg.Add(1)
							go func(t *tribe) {
								defer wg.Done()
								for {
									if t.agreements != nil {
										if _, ok := t.agreements["clan1"]; ok {
											c.So(ok, ShouldEqual, true)
											return
										}
									}
									time.Sleep(50 * time.Millisecond)
								}
							}(t)
						}
						wg.Wait()

						numAddMessages := 10
						Convey(fmt.Sprintf("Handles %d plugin 'add messages' broadcasted across the cluster", numAddMessages), func() {
							for i := 0; i < numAddMessages; i++ {
								tribes[0].AddPlugin("clan1", agreement.Plugin{Name_: fmt.Sprintf("plugin%v", i), Version_: 1})
								// time.Sleep(time.Millisecond * 50)
							}
							wg := sync.WaitGroup{}
							for _, tr := range tribes {
								wg.Add(1)
								go func(tr *tribe) {
									defer wg.Done()
									for {
										if clan, ok := tr.agreements["clan1"]; ok {
											if len(clan.PluginAgreement.Plugins) == numAddMessages {
												return
											}
											time.Sleep(50 * time.Millisecond)
											log.Debugf("%v has %v of %v plugins and %d intents\n", tr.memberlist.LocalNode().Name, len(clan.PluginAgreement.Plugins), numAddMessages, len(tr.intentBuffer))
										}
									}
								}(tr)
							}
							log.Debugf("Waits for %d members of clan1 to have %d plugins\n", numOfTribes, numAddMessages)
							wg.Wait()
							for i := 0; i < numOfTribes; i++ {
								So(len(tribes[i].agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, numAddMessages)
								logger.Debugf("%v has %v intents\n", tribes[i].memberlist.LocalNode().Name, len(tribes[i].intentBuffer))
								So(len(tribes[i].intentBuffer), ShouldEqual, 0)
								for k, v := range tribes[i].intentBuffer {
									logger.Debugf("\tadd intent %v %v\n", k, v)

								}
							}

							Convey("Handles duplicate 'add plugin' messages", func() {
								t := tribes[rand.Intn(numOfTribes)]
								msg := &pluginMsg{
									Plugin: agreement.Plugin{
										Name_:    "pluginABC",
										Version_: 1,
									},
									UUID:          uuid.New(),
									AgreementName: "clan1",
									LTime:         t.clock.Time(),
									Type:          addPluginMsgType,
								}
								So(len(t.intentBuffer), ShouldEqual, 0)
								t.handleAddPlugin(msg)
								before := len(t.agreements["clan1"].PluginAgreement.Plugins)
								t.handleAddPlugin(msg)
								after := len(t.agreements["clan1"].PluginAgreement.Plugins)
								So(before, ShouldEqual, after)

								Convey("Handles out-of-order 'add plugin' messages", func() {
									msg := &pluginMsg{
										Plugin: agreement.Plugin{
											Name_:    "pluginABC",
											Version_: 1,
										},
										UUID:          uuid.New(),
										AgreementName: "clan1",
										LTime:         t.clock.Time(),
										Type:          addPluginMsgType,
									}
									t.handleAddPlugin(msg)
									So(len(t.intentBuffer), ShouldEqual, 1)

									Convey("Handles duplicate out-of-order 'add plugin' messages", func() {
										before := len(t.agreements["clan1"].PluginAgreement.Plugins)
										t.handleAddPlugin(msg)
										after := len(t.agreements["clan1"].PluginAgreement.Plugins)
										So(before, ShouldEqual, after)
										So(len(t.intentBuffer), ShouldEqual, 1)
										So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldBeGreaterThan, numAddMessages)
										t.handleRemovePlugin(&pluginMsg{
											LTime:         t.clock.Time(),
											Plugin:        agreement.Plugin{Name_: "pluginABC", Version_: 1},
											AgreementName: "clan1",
											Type:          removePluginMsgType,
										})
										So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldBeGreaterThan, numAddMessages)
										So(len(t.intentBuffer), ShouldEqual, 0)

										// removes the plugin added to test duplicates
										t.handleRemovePlugin(&pluginMsg{
											LTime:         t.clock.Time(),
											Plugin:        agreement.Plugin{Name_: "pluginABC", Version_: 1},
											AgreementName: "clan1",
											Type:          removePluginMsgType,
										})
										So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, numAddMessages)
										// wait for all members of the tribe to get back to 10 plugins
										wg = sync.WaitGroup{}
										for _, tr := range tribes {
											wg.Add(1)
											go func(tr *tribe) {
												defer wg.Done()
												for {
													select {
													case <-time.After(1500 * time.Millisecond):
														c.So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, numAddMessages)
													default:
														if clan, ok := tr.agreements["clan1"]; ok {
															if len(clan.PluginAgreement.Plugins) == numAddMessages {
																return
															}
															time.Sleep(50 * time.Millisecond)
														}
													}
												}
											}(tr)
										}

										Convey("Handles a 'remove plugin' messages broadcasted across the cluster", func(c C) {
											for _, t := range tribes {
												So(len(t.intentBuffer), ShouldEqual, 0)
												So(len(t.intentBuffer), ShouldEqual, 0)
												So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, numAddMessages)
											}
											t := tribes[rand.Intn(numOfTribes)]
											plugin := t.agreements["clan1"].PluginAgreement.Plugins[rand.Intn(numAddMessages)]
											before := len(t.agreements["clan1"].PluginAgreement.Plugins)
											t.RemovePlugin("clan1", plugin)
											after := len(t.agreements["clan1"].PluginAgreement.Plugins)
											So(before-after, ShouldEqual, 1)
											var wg sync.WaitGroup
											for _, t := range tribes {
												wg.Add(1)
												go func(t *tribe) {
													defer wg.Done()
													for {
														select {
														case <-time.After(1500 * time.Millisecond):
															c.So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, after)
														default:
															if len(t.agreements["clan1"].PluginAgreement.Plugins) == after {
																c.So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, after)
																return
															}
															time.Sleep(50 * time.Millisecond)
														}
													}
												}(t)
											}
											wg.Done()

											Convey("Handles out-of-order remove", func() {
												t := tribes[rand.Intn(numOfTribes)]
												plugin := t.agreements["clan1"].PluginAgreement.Plugins[rand.Intn(numAddMessages-1)]
												msg := &pluginMsg{
													LTime:         t.clock.Increment(),
													Plugin:        plugin,
													AgreementName: "clan1",
													UUID:          uuid.New(),
													Type:          removePluginMsgType,
												}
												before := len(t.agreements["clan1"].PluginAgreement.Plugins)
												t.handleRemovePlugin(msg)
												So(before-1, ShouldEqual, len(t.agreements["clan1"].PluginAgreement.Plugins))
												before = len(t.agreements["clan1"].PluginAgreement.Plugins)
												msg.UUID = uuid.New()
												msg.LTime = t.clock.Increment()
												t.handleRemovePlugin(msg)
												after := len(t.agreements["clan1"].PluginAgreement.Plugins)
												So(before, ShouldEqual, after)
												So(len(t.intentBuffer), ShouldEqual, 1)

												Convey("Handles duplicate out-of-order 'remove plugin' messages", func() {
													t.handleRemovePlugin(msg)
													after := len(t.agreements["clan1"].PluginAgreement.Plugins)
													So(before, ShouldEqual, after)
													So(len(t.intentBuffer), ShouldEqual, 1)

													t.handleAddPlugin(&pluginMsg{
														LTime:         t.clock.Increment(),
														Plugin:        plugin,
														AgreementName: "clan1",
														Type:          addPluginMsgType,
													})
													So(len(t.intentBuffer), ShouldEqual, 0)
													ok, _ := t.agreements["clan1"].PluginAgreement.Plugins.Contains(msg.Plugin)
													So(ok, ShouldBeFalse)

													Convey("Handles old 'remove plugin' messages", func() {
														t := tribes[rand.Intn(numOfTribes)]
														plugin := t.agreements["clan1"].PluginAgreement.Plugins[rand.Intn(len(t.agreements["clan1"].PluginAgreement.Plugins))]
														msg := &pluginMsg{
															LTime:         LTime(1025),
															Plugin:        plugin,
															AgreementName: "clan1",
															UUID:          uuid.New(),
															Type:          removePluginMsgType,
														}
														before := len(t.agreements["clan1"].PluginAgreement.Plugins)
														t.handleRemovePlugin(msg)
														after := len(t.agreements["clan1"].PluginAgreement.Plugins)
														So(before-1, ShouldEqual, after)
														msg2 := &pluginMsg{
															LTime:         LTime(513),
															Plugin:        plugin,
															AgreementName: "clan1",
															UUID:          uuid.New(),
															Type:          addPluginMsgType,
														}
														before = len(t.agreements["clan1"].PluginAgreement.Plugins)
														t.handleAddPlugin(msg2)
														after = len(t.agreements["clan1"].PluginAgreement.Plugins)
														So(before, ShouldEqual, after)
														msg3 := &pluginMsg{
															LTime:         LTime(513),
															Plugin:        plugin,
															AgreementName: "clan1",
															UUID:          uuid.New(),
															Type:          removePluginMsgType,
														}
														before = len(t.agreements["clan1"].PluginAgreement.Plugins)
														t.handleRemovePlugin(msg3)
														after = len(t.agreements["clan1"].PluginAgreement.Plugins)
														So(before, ShouldEqual, after)

														Convey("The tribe agrees on plugin agreements", func(c C) {
															var wg sync.WaitGroup
															for _, t := range tribes {
																wg.Add(1)
																go func(t *tribe) {
																	for {
																		defer wg.Done()
																		select {
																		case <-time.After(1 * time.Second):
																			c.So(len(t.memberlist.Members()), ShouldEqual, numOfTribes)
																		default:
																			if len(t.agreements["clan1"].PluginAgreement.Plugins) == numAddMessages-1 {
																				c.So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, numAddMessages-1)
																				return
																			}
																		}

																	}
																}(t)
															}
															wg.Done()
														})
													})
												})
											})
										})
									})
								})
							})
						})
					})
				})
			})
		})
	})
}
Esempio n. 2
0
func TestTribeAgreements(t *testing.T) {
	numOfTribes := 5
	tribes := getTribes(numOfTribes, nil)
	Convey(fmt.Sprintf("%d tribes are started", numOfTribes), t, func() {
		for i := 0; i < numOfTribes; i++ {
			log.Debugf("%v is reporting %v members", i, len(tribes[i].memberlist.Members()))
			So(len(tribes[0].memberlist.Members()), ShouldEqual, len(tribes[i].memberlist.Members()))
		}
		Convey("The cluster agrees on membership", func() {
			for i := 0; i < numOfTribes; i++ {

				So(
					len(tribes[0].memberlist.Members()),
					ShouldEqual,
					len(tribes[i].memberlist.Members()),
				)
				So(len(tribes[0].members), ShouldEqual, len(tribes[i].members))
			}

			Convey("A member handles", func() {
				agreementName := "agreement1"
				t := tribes[0]
				t2 := tribes[1]
				Convey("an out-of-order join agreement message", func() {
					msg := &agreementMsg{
						LTime:         t.clock.Increment(),
						UUID:          uuid.New(),
						AgreementName: agreementName,
						MemberName:    t.memberlist.LocalNode().Name,
						Type:          joinAgreementMsgType,
					}
					msg2 := &agreementMsg{
						LTime:         t.clock.Increment(),
						UUID:          uuid.New(),
						AgreementName: agreementName,
						MemberName:    t2.memberlist.LocalNode().Name,
						Type:          joinAgreementMsgType,
					}

					b := t.handleJoinAgreement(msg)
					So(b, ShouldEqual, true)
					So(len(t.intentBuffer), ShouldEqual, 1)
					t.broadcast(joinAgreementMsgType, msg, nil)
					timer := time.After(2 * time.Second)
				loop1:
					for {
						select {
						case <-timer:
							So("Timed out", ShouldEqual, "")
						default:
							if len(t2.intentBuffer) > 0 {
								break loop1
							}
						}
					}
					So(len(t2.intentBuffer), ShouldEqual, 1)

					b = t.handleJoinAgreement(msg2)
					So(b, ShouldEqual, true)
					So(len(t.intentBuffer), ShouldEqual, 2)
					t.broadcast(joinAgreementMsgType, msg2, nil)

					timer = time.After(2 * time.Second)
				loop2:
					for {
						select {
						case <-timer:
							So("Timed out", ShouldEqual, "")
						default:
							if len(t2.intentBuffer) == 2 {
								break loop2
							}
						}
					}
					So(len(t2.intentBuffer), ShouldEqual, 2)

					Convey("an out-of-order add plugin message", func() {
						plugin := agreement.Plugin{Name_: "plugin1", Version_: 1}
						msg := &pluginMsg{
							LTime:         t.clock.Increment(),
							UUID:          uuid.New(),
							Plugin:        plugin,
							AgreementName: agreementName,
							Type:          addPluginMsgType,
						}
						b := t.handleAddPlugin(msg)
						So(b, ShouldEqual, true)
						So(len(t.intentBuffer), ShouldEqual, 3)
						t.broadcast(addPluginMsgType, msg, nil)

						Convey("an add agreement", func() {
							err := t.AddAgreement(agreementName)
							So(err, ShouldBeNil)
							err = t.AddAgreement(agreementName)
							So(err.Error(), ShouldResemble, errAgreementAlreadyExists.Error())
							var wg sync.WaitGroup
							for _, t := range tribes {
								wg.Add(1)
								go func(t *tribe) {
									defer wg.Done()
									for {
										if a, ok := t.agreements[agreementName]; ok {
											logger.Debugf("%s has %d plugins in agreement '%s' and %d intents", t.memberlist.LocalNode().Name, len(t.agreements[agreementName].PluginAgreement.Plugins), agreementName, len(t.intentBuffer))
											if ok, _ := a.PluginAgreement.Plugins.Contains(plugin); ok {
												if len(t.intentBuffer) == 0 {
													return
												}
											}
										}
										logger.Debugf("%s has %d intents", t.memberlist.LocalNode().Name, len(t.intentBuffer))
										time.Sleep(50 * time.Millisecond)
									}
								}(t)
							}
							wg.Wait()

							Convey("being added to an agreement it already belongs to", func() {
								err := t.JoinAgreement(agreementName, t.memberlist.LocalNode().Name)
								So(err.Error(), ShouldResemble, errAlreadyMemberOfPluginAgreement.Error())

								Convey("leaving an agreement that doesn't exist", func() {
									err := t.LeaveAgreement("whatever", t.memberlist.LocalNode().Name)
									So(err.Error(), ShouldResemble, errAgreementDoesNotExist.Error())

									Convey("an unknown member trying to leave an agreement", func() {
										err := t.LeaveAgreement(agreementName, "whatever")
										So(err.Error(), ShouldResemble, errUnknownMember.Error())

										Convey("a member leaving an agreement it isn't part of", func() {
											err := t.LeaveAgreement(agreementName, tribes[2].memberlist.LocalNode().Name)
											So(err, ShouldNotBeNil)
											So(err.Error(), ShouldResemble, errNotAMember.Error())

											Convey("an unknown member trying to join an agreement", func() {
												msg := &agreementMsg{
													LTime:         t.clock.Time(),
													UUID:          uuid.New(),
													AgreementName: agreementName,
													MemberName:    "whatever",
													Type:          joinAgreementMsgType,
												}
												err := t.joinAgreement(msg)
												So(err, ShouldNotBeNil)
												So(err.Error(), ShouldResemble, errUnknownMember.Error())

												Convey("leaving an agreement", func() {
													So(len(t.agreements[agreementName].Members), ShouldEqual, 2)
													So(t.members[t.memberlist.LocalNode().Name].PluginAgreement, ShouldNotBeNil)
													err := t.LeaveAgreement(agreementName, t.memberlist.LocalNode().Name)
													So(err, ShouldBeNil)
													So(len(t.agreements[agreementName].Members), ShouldEqual, 1)
													So(t.members[t.memberlist.LocalNode().Name].PluginAgreement, ShouldBeNil)
													Convey("leaving a tribe results in the member leaving the agreement", func() {
														t2.memberlist.Leave(500 * time.Millisecond)
														timer := time.After(2 * time.Second)
													loop3:
														for {
															select {
															case <-timer:
																So("Timed out", ShouldEqual, "")
															default:
																if len(t.agreements[agreementName].Members) == 0 {
																	break loop3
																}
															}
														}
														So(len(t.agreements[agreementName].Members), ShouldEqual, 0)
														Convey("removes an agreement", func() {
															err := t.RemoveAgreement(agreementName)
															So(err, ShouldBeNil)
															So(len(t.agreements), ShouldEqual, 0)
															Convey("removes an agreement that no longer exists", func() {
																err := t.RemoveAgreement(agreementName)
																So(err.Error(), ShouldResemble, errAgreementDoesNotExist.Error())
																So(len(t.agreements), ShouldEqual, 0)
															})
														})
													})
												})
											})
										})
									})
								})
							})
						})
					})
				})
			})
		})
	})
}