コード例 #1
0
ファイル: tribe_test.go プロジェクト: IRCody/snap
func TestTribePluginAgreement(t *testing.T) {
	numOfTribes := 5
	// tribePort := 52600
	tribes := getTribes(numOfTribes, nil)
	Convey(fmt.Sprintf("%d tribes are started", numOfTribes), t, func() {
		for i := 0; i < numOfTribes; i++ {
			So(
				len(tribes[0].memberlist.Members()),
				ShouldEqual,
				len(tribes[i].memberlist.Members()),
			)
			logger.Debugf("%v has %v members", tribes[i].memberlist.LocalNode().Name, len(tribes[i].members))
			So(len(tribes[i].members), ShouldEqual, numOfTribes)
		}

		Convey("The cluster agrees on membership", func() {
			for i := 0; i < numOfTribes; i++ {
				log.Debugf("%v is reporting %v members", i, len(tribes[i].memberlist.Members()))
				So(len(tribes[0].memberlist.Members()), ShouldEqual, len(tribes[i].memberlist.Members()))
				So(len(tribes[0].members), ShouldEqual, len(tribes[i].members))
			}
			oldMember := tribes[0]
			err := tribes[0].memberlist.Leave(2 * time.Second)
			// err := tribes[0].memberlist.Shutdown()
			So(err, ShouldBeNil)
			tribes = append(tribes[:0], tribes[1:]...)

			Convey("Membership decreases as members leave", func(c C) {
				wg := sync.WaitGroup{}
				for i := range tribes {
					wg.Add(1)
					go func(i int) {
						defer wg.Done()
						for {
							if len(tribes[i].members) == len(tribes) {
								c.So(len(tribes[i].members), ShouldEqual, len(tribes))
								return
							}
							time.Sleep(20 * time.Millisecond)
						}
					}(i)
				}
				wg.Wait()
				err := oldMember.memberlist.Shutdown()
				So(err, ShouldBeNil)
				So(len(tribes[rand.Intn(len(tribes))].memberlist.Members()), ShouldEqual, len(tribes))
				So(len(tribes[1].members), ShouldEqual, len(tribes))

				Convey("Membership increases as members join", func(c C) {
					seed := fmt.Sprintf("%v:%v", tribes[0].memberlist.LocalNode().Addr, tribes[0].memberlist.LocalNode().Port)
					conf := getTestConfig()
					conf.Name = fmt.Sprintf("member-%d", numOfTribes+1)
					conf.Seed = seed
					tr, err := New(conf)
					if err != nil {
						So(err, ShouldBeNil)
					}
					tribes = append(tribes, tr)

					wg := sync.WaitGroup{}
					for i := range tribes {
						wg.Add(1)
						go func(i int) {
							defer wg.Done()
							for {
								if len(tribes[i].memberlist.Members()) == len(tribes) {
									c.So(len(tribes[i].members), ShouldEqual, len(tribes))
									return
								}
								time.Sleep(20 * time.Millisecond)
							}
						}(i)
					}
					wg.Wait()
					So(len(tribes[rand.Intn(len(tribes))].memberlist.Members()), ShouldEqual, len(tribes))
					So(len(tribes[rand.Intn(len(tribes))].members), ShouldEqual, len(tribes))

					Convey("Handles a 'add agreement' message broadcasted across the cluster", func(c C) {
						tribes[0].AddAgreement("clan1")
						var wg sync.WaitGroup
						for _, t := range tribes {
							wg.Add(1)
							go func(t *tribe) {
								defer wg.Done()
								for {
									if t.agreements != nil {
										if _, ok := t.agreements["clan1"]; ok {
											c.So(ok, ShouldEqual, true)
											return
										}
									}
									time.Sleep(50 * time.Millisecond)
								}
							}(t)
						}
						wg.Wait()

						numAddMessages := 10
						Convey(fmt.Sprintf("Handles %d plugin 'add messages' broadcasted across the cluster", numAddMessages), func() {
							for i := 0; i < numAddMessages; i++ {
								tribes[0].AddPlugin("clan1", agreement.Plugin{Name_: fmt.Sprintf("plugin%v", i), Version_: 1})
								// time.Sleep(time.Millisecond * 50)
							}
							wg := sync.WaitGroup{}
							for _, tr := range tribes {
								wg.Add(1)
								go func(tr *tribe) {
									defer wg.Done()
									for {
										if clan, ok := tr.agreements["clan1"]; ok {
											if len(clan.PluginAgreement.Plugins) == numAddMessages {
												return
											}
											time.Sleep(50 * time.Millisecond)
											log.Debugf("%v has %v of %v plugins and %d intents\n", tr.memberlist.LocalNode().Name, len(clan.PluginAgreement.Plugins), numAddMessages, len(tr.intentBuffer))
										}
									}
								}(tr)
							}
							log.Debugf("Waits for %d members of clan1 to have %d plugins\n", numOfTribes, numAddMessages)
							wg.Wait()
							for i := 0; i < numOfTribes; i++ {
								So(len(tribes[i].agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, numAddMessages)
								logger.Debugf("%v has %v intents\n", tribes[i].memberlist.LocalNode().Name, len(tribes[i].intentBuffer))
								So(len(tribes[i].intentBuffer), ShouldEqual, 0)
								for k, v := range tribes[i].intentBuffer {
									logger.Debugf("\tadd intent %v %v\n", k, v)

								}
							}

							Convey("Handles duplicate 'add plugin' messages", func() {
								t := tribes[rand.Intn(numOfTribes)]
								msg := &pluginMsg{
									Plugin: agreement.Plugin{
										Name_:    "pluginABC",
										Version_: 1,
									},
									UUID:          uuid.New(),
									AgreementName: "clan1",
									LTime:         t.clock.Time(),
									Type:          addPluginMsgType,
								}
								So(len(t.intentBuffer), ShouldEqual, 0)
								t.handleAddPlugin(msg)
								before := len(t.agreements["clan1"].PluginAgreement.Plugins)
								t.handleAddPlugin(msg)
								after := len(t.agreements["clan1"].PluginAgreement.Plugins)
								So(before, ShouldEqual, after)

								Convey("Handles out-of-order 'add plugin' messages", func() {
									msg := &pluginMsg{
										Plugin: agreement.Plugin{
											Name_:    "pluginABC",
											Version_: 1,
										},
										UUID:          uuid.New(),
										AgreementName: "clan1",
										LTime:         t.clock.Time(),
										Type:          addPluginMsgType,
									}
									t.handleAddPlugin(msg)
									So(len(t.intentBuffer), ShouldEqual, 1)

									Convey("Handles duplicate out-of-order 'add plugin' messages", func() {
										before := len(t.agreements["clan1"].PluginAgreement.Plugins)
										t.handleAddPlugin(msg)
										after := len(t.agreements["clan1"].PluginAgreement.Plugins)
										So(before, ShouldEqual, after)
										So(len(t.intentBuffer), ShouldEqual, 1)
										So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldBeGreaterThan, numAddMessages)
										t.handleRemovePlugin(&pluginMsg{
											LTime:         t.clock.Time(),
											Plugin:        agreement.Plugin{Name_: "pluginABC", Version_: 1},
											AgreementName: "clan1",
											Type:          removePluginMsgType,
										})
										So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldBeGreaterThan, numAddMessages)
										So(len(t.intentBuffer), ShouldEqual, 0)

										// removes the plugin added to test duplicates
										t.handleRemovePlugin(&pluginMsg{
											LTime:         t.clock.Time(),
											Plugin:        agreement.Plugin{Name_: "pluginABC", Version_: 1},
											AgreementName: "clan1",
											Type:          removePluginMsgType,
										})
										So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, numAddMessages)
										// wait for all members of the tribe to get back to 10 plugins
										wg = sync.WaitGroup{}
										for _, tr := range tribes {
											wg.Add(1)
											go func(tr *tribe) {
												defer wg.Done()
												for {
													select {
													case <-time.After(1500 * time.Millisecond):
														c.So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, numAddMessages)
													default:
														if clan, ok := tr.agreements["clan1"]; ok {
															if len(clan.PluginAgreement.Plugins) == numAddMessages {
																return
															}
															time.Sleep(50 * time.Millisecond)
														}
													}
												}
											}(tr)
										}

										Convey("Handles a 'remove plugin' messages broadcasted across the cluster", func(c C) {
											for _, t := range tribes {
												So(len(t.intentBuffer), ShouldEqual, 0)
												So(len(t.intentBuffer), ShouldEqual, 0)
												So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, numAddMessages)
											}
											t := tribes[rand.Intn(numOfTribes)]
											plugin := t.agreements["clan1"].PluginAgreement.Plugins[rand.Intn(numAddMessages)]
											before := len(t.agreements["clan1"].PluginAgreement.Plugins)
											t.RemovePlugin("clan1", plugin)
											after := len(t.agreements["clan1"].PluginAgreement.Plugins)
											So(before-after, ShouldEqual, 1)
											var wg sync.WaitGroup
											for _, t := range tribes {
												wg.Add(1)
												go func(t *tribe) {
													defer wg.Done()
													for {
														select {
														case <-time.After(1500 * time.Millisecond):
															c.So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, after)
														default:
															if len(t.agreements["clan1"].PluginAgreement.Plugins) == after {
																c.So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, after)
																return
															}
															time.Sleep(50 * time.Millisecond)
														}
													}
												}(t)
											}
											wg.Done()

											Convey("Handles out-of-order remove", func() {
												t := tribes[rand.Intn(numOfTribes)]
												plugin := t.agreements["clan1"].PluginAgreement.Plugins[rand.Intn(numAddMessages-1)]
												msg := &pluginMsg{
													LTime:         t.clock.Increment(),
													Plugin:        plugin,
													AgreementName: "clan1",
													UUID:          uuid.New(),
													Type:          removePluginMsgType,
												}
												before := len(t.agreements["clan1"].PluginAgreement.Plugins)
												t.handleRemovePlugin(msg)
												So(before-1, ShouldEqual, len(t.agreements["clan1"].PluginAgreement.Plugins))
												before = len(t.agreements["clan1"].PluginAgreement.Plugins)
												msg.UUID = uuid.New()
												msg.LTime = t.clock.Increment()
												t.handleRemovePlugin(msg)
												after := len(t.agreements["clan1"].PluginAgreement.Plugins)
												So(before, ShouldEqual, after)
												So(len(t.intentBuffer), ShouldEqual, 1)

												Convey("Handles duplicate out-of-order 'remove plugin' messages", func() {
													t.handleRemovePlugin(msg)
													after := len(t.agreements["clan1"].PluginAgreement.Plugins)
													So(before, ShouldEqual, after)
													So(len(t.intentBuffer), ShouldEqual, 1)

													t.handleAddPlugin(&pluginMsg{
														LTime:         t.clock.Increment(),
														Plugin:        plugin,
														AgreementName: "clan1",
														Type:          addPluginMsgType,
													})
													So(len(t.intentBuffer), ShouldEqual, 0)
													ok, _ := t.agreements["clan1"].PluginAgreement.Plugins.Contains(msg.Plugin)
													So(ok, ShouldBeFalse)

													Convey("Handles old 'remove plugin' messages", func() {
														t := tribes[rand.Intn(numOfTribes)]
														plugin := t.agreements["clan1"].PluginAgreement.Plugins[rand.Intn(len(t.agreements["clan1"].PluginAgreement.Plugins))]
														msg := &pluginMsg{
															LTime:         LTime(1025),
															Plugin:        plugin,
															AgreementName: "clan1",
															UUID:          uuid.New(),
															Type:          removePluginMsgType,
														}
														before := len(t.agreements["clan1"].PluginAgreement.Plugins)
														t.handleRemovePlugin(msg)
														after := len(t.agreements["clan1"].PluginAgreement.Plugins)
														So(before-1, ShouldEqual, after)
														msg2 := &pluginMsg{
															LTime:         LTime(513),
															Plugin:        plugin,
															AgreementName: "clan1",
															UUID:          uuid.New(),
															Type:          addPluginMsgType,
														}
														before = len(t.agreements["clan1"].PluginAgreement.Plugins)
														t.handleAddPlugin(msg2)
														after = len(t.agreements["clan1"].PluginAgreement.Plugins)
														So(before, ShouldEqual, after)
														msg3 := &pluginMsg{
															LTime:         LTime(513),
															Plugin:        plugin,
															AgreementName: "clan1",
															UUID:          uuid.New(),
															Type:          removePluginMsgType,
														}
														before = len(t.agreements["clan1"].PluginAgreement.Plugins)
														t.handleRemovePlugin(msg3)
														after = len(t.agreements["clan1"].PluginAgreement.Plugins)
														So(before, ShouldEqual, after)

														Convey("The tribe agrees on plugin agreements", func(c C) {
															var wg sync.WaitGroup
															for _, t := range tribes {
																wg.Add(1)
																go func(t *tribe) {
																	for {
																		defer wg.Done()
																		select {
																		case <-time.After(1 * time.Second):
																			c.So(len(t.memberlist.Members()), ShouldEqual, numOfTribes)
																		default:
																			if len(t.agreements["clan1"].PluginAgreement.Plugins) == numAddMessages-1 {
																				c.So(len(t.agreements["clan1"].PluginAgreement.Plugins), ShouldEqual, numAddMessages-1)
																				return
																			}
																		}

																	}
																}(t)
															}
															wg.Done()
														})
													})
												})
											})
										})
									})
								})
							})
						})
					})
				})
			})
		})
	})
}