func (s *TestSuite) TestBeginMaintenance(c *C) { clearTestMaintenance() _, _ = inst.ReadTopologyInstance(&masterKey) _, err := inst.BeginMaintenance(&masterKey, "unittest", "TestBeginMaintenance") c.Assert(err, IsNil) }
func (s *TestSuite) TestReadTopologyAndInstanceSlave(c *C) { i, _ := inst.ReadTopologyInstance(&slave1Key) iRead, found, _ := inst.ReadInstance(&slave1Key) c.Assert(found, Equals, true) c.Assert(iRead.Key.Hostname, Equals, i.Key.Hostname) c.Assert(iRead.Version, Equals, i.Version) }
func (s *TestSuite) TestReadTopologySlave(c *C) { key := slave1Key i, _ := inst.ReadTopologyInstance(&key) c.Assert(i.Key.Hostname, Equals, key.Hostname) c.Assert(i.IsSlave(), Equals, true) c.Assert(len(i.SlaveHosts), Equals, 0) }
func (s *TestSuite) TestForgetMaster(c *C) { _, _ = inst.ReadTopologyInstance(&masterKey) _, found, _ := inst.ReadInstance(&masterKey) c.Assert(found, Equals, true) inst.ForgetInstance(&masterKey) _, found, _ = inst.ReadInstance(&masterKey) c.Assert(found, Equals, false) }
func (s *TestSuite) TestGetMasterOfASlave(c *C) { i, err := inst.ReadTopologyInstance(&slave1Key) c.Assert(err, IsNil) master, err := inst.GetInstanceMaster(i) c.Assert(err, IsNil) c.Assert(master.IsSlave(), Equals, false) c.Assert(master.Key.Port, Equals, 22987) }
func (s *TestSuite) TestReadTopologyAndInstanceMaster(c *C) { i, _ := inst.ReadTopologyInstance(&masterKey) iRead, found, _ := inst.ReadInstance(&masterKey) c.Assert(found, Equals, true) c.Assert(iRead.Key.Hostname, Equals, i.Key.Hostname) c.Assert(iRead.Version, Equals, i.Version) c.Assert(len(iRead.SlaveHosts), Equals, len(i.SlaveHosts)) }
func (s *TestSuite) TestReadTopologyUnexisting(c *C) { key := inst.InstanceKey{ Hostname: "127.0.0.1", Port: 22999, } _, err := inst.ReadTopologyInstance(&key) c.Assert(err, Not(IsNil)) }
func (s *TestSuite) TestMakeCoMasterAndBack(c *C) { clearTestMaintenance() slave1, err := inst.MakeCoMaster(&slave1Key) c.Assert(err, IsNil) // Now master & slave1 expected to be co-masters. Check! master, _ := inst.ReadTopologyInstance(&masterKey) c.Assert(master.MasterKey.Port, Not(Equals), inst.InvalidPort) c.Assert(master.IsSlaveOf(slave1), Equals, true) c.Assert(slave1.IsSlaveOf(master), Equals, true) // detach - resotre to original state master, err = inst.DetachSlaveFromMaster(&masterKey) slave1, _ = inst.ReadTopologyInstance(&slave1Key) c.Assert(err, IsNil) c.Assert(master.MasterKey.Port, Equals, inst.InvalidPort) }
func (s *TestSuite) TestReadTopologyMaster(c *C) { key := masterKey i, _ := inst.ReadTopologyInstance(&key) c.Assert(i.Key.Hostname, Equals, key.Hostname) c.Assert(i.IsSlave(), Equals, false) c.Assert(len(i.SlaveHosts), Equals, 3) c.Assert(len(i.SlaveHosts.GetInstanceKeys()), Equals, len(i.SlaveHosts)) }
func (s *TestSuite) TestFailEndMaintenanceTwice(c *C) { clearTestMaintenance() _, _ = inst.ReadTopologyInstance(&masterKey) k, err := inst.BeginMaintenance(&masterKey, "unittest", "TestFailEndMaintenanceTwice") c.Assert(err, IsNil) err = inst.EndMaintenance(k) c.Assert(err, IsNil) err = inst.EndMaintenance(k) c.Assert(err, Not(IsNil)) }
func (s *TestSuite) TestFailMoveBelowUponMaintenance(c *C) { clearTestMaintenance() _, _ = inst.ReadTopologyInstance(&slave1Key) k, err := inst.BeginMaintenance(&slave1Key, "unittest", "TestBeginEndMaintenance") c.Assert(err, IsNil) _, err = inst.MoveBelow(&slave1Key, &slave2Key) c.Assert(err, Not(IsNil)) err = inst.EndMaintenance(k) c.Assert(err, IsNil) }
func (s *TestSuite) TestStopStartSlave(c *C) { i, _ := inst.ReadTopologyInstance(&slave1Key) c.Assert(i.SlaveRunning(), Equals, true) i, _ = inst.StopSlaveNicely(&i.Key) c.Assert(i.SlaveRunning(), Equals, false) c.Assert(i.SQLThreadUpToDate(), Equals, true) i, _ = inst.StartSlave(&i.Key) c.Assert(i.SlaveRunning(), Equals, true) }
func (s *TestSuite) TestFailMoveBelowUponOtherSlaveStopped(c *C) { clearTestMaintenance() slave1, _ := inst.ReadTopologyInstance(&slave1Key) c.Assert(slave1.SlaveRunning(), Equals, true) slave1, _ = inst.StopSlaveNicely(&slave1.Key) c.Assert(slave1.SlaveRunning(), Equals, false) _, err := inst.MoveBelow(&slave2Key, &slave1Key) c.Assert(err, Not(IsNil)) _, _ = inst.StartSlave(&slave1.Key) }
func (s *TestSuite) TestMoveBelowAndBackComplex(c *C) { clearTestMaintenance() // become child slave1, _ := inst.MoveBelow(&slave1Key, &slave2Key) c.Assert(slave1.MasterKey.Equals(&slave2Key), Equals, true) c.Assert(slave1.SlaveRunning(), Equals, true) // Now let's have fun. Stop slave2 (which is now parent of slave1), execute queries on master, // move s1 back under master, start all, verify queries. _, err := inst.StopSlave(&slave2Key) c.Assert(err, IsNil) randValue := rand.Int() _, err = inst.ExecInstance(&masterKey, `replace into orchestrator_test.test_table (name, value) values ('TestMoveBelowAndBackComplex', ?)`, randValue) c.Assert(err, IsNil) master, err := inst.ReadTopologyInstance(&masterKey) c.Assert(err, IsNil) // And back; keep topology intact slave1, err = inst.MoveUp(&slave1Key) c.Assert(err, IsNil) _, err = inst.MasterPosWait(&slave1Key, &master.SelfBinlogCoordinates) c.Assert(err, IsNil) slave2, err := inst.ReadTopologyInstance(&slave2Key) c.Assert(err, IsNil) _, err = inst.MasterPosWait(&slave2Key, &master.SelfBinlogCoordinates) c.Assert(err, IsNil) // Now check for value! var value1, value2 int inst.ScanInstanceRow(&slave1Key, `select value from orchestrator_test.test_table where name='TestMoveBelowAndBackComplex'`, &value1) inst.ScanInstanceRow(&slave2Key, `select value from orchestrator_test.test_table where name='TestMoveBelowAndBackComplex'`, &value2) c.Assert(inst.InstancesAreSiblings(slave1, slave2), Equals, true) c.Assert(value1, Equals, randValue) c.Assert(value2, Equals, randValue) }
func (s *TestSuite) TestDiscover(c *C) { var err error _, err = db.ExecOrchestrator("delete from database_instance where hostname = ? and port = ?", masterKey.Hostname, masterKey.Port) _, err = db.ExecOrchestrator("delete from database_instance where hostname = ? and port = ?", slave1Key.Hostname, slave1Key.Port) _, err = db.ExecOrchestrator("delete from database_instance where hostname = ? and port = ?", slave2Key.Hostname, slave2Key.Port) _, err = db.ExecOrchestrator("delete from database_instance where hostname = ? and port = ?", slave3Key.Hostname, slave3Key.Port) _, found, _ := inst.ReadInstance(&masterKey) c.Assert(found, Equals, false) _, _ = inst.ReadTopologyInstance(&slave1Key) orchestrator.StartDiscovery(slave1Key) _, found, err = inst.ReadInstance(&slave1Key) c.Assert(found, Equals, true) c.Assert(err, IsNil) }
func (s *TestSuite) TestMoveBelowAndBack(c *C) { clearTestMaintenance() // become child slave1, err := inst.MoveBelow(&slave1Key, &slave2Key) c.Assert(err, IsNil) c.Assert(slave1.MasterKey.Equals(&slave2Key), Equals, true) c.Assert(slave1.SlaveRunning(), Equals, true) // And back; keep topology intact slave1, _ = inst.MoveUp(&slave1Key) slave2, _ := inst.ReadTopologyInstance(&slave2Key) c.Assert(inst.InstancesAreSiblings(slave1, slave2), Equals, true) c.Assert(slave1.SlaveRunning(), Equals, true) }
// DiscoverInstance will attempt discovering an instance (unless it is already up to date) and will // list down its master and slaves (if any) for further discovery. func DiscoverInstance(instanceKey inst.InstanceKey) { instanceKey.Formalize() if !instanceKey.IsValid() { return } instance, found, err := inst.ReadInstance(&instanceKey) if found && instance.IsUpToDate && instance.IsLastCheckValid { // we've already discovered this one. Skip! goto Cleanup } // First we've ever heard of this instance. Continue investigation: instance, err = inst.ReadTopologyInstance(&instanceKey) // panic can occur (IO stuff). Therefore it may happen // that instance is nil. Check it. if err != nil || instance == nil { goto Cleanup } fmt.Printf("host: %+v, master: %+v\n", instance.Key, instance.MasterKey) // Investigate slaves: for _, slaveKey := range instance.SlaveHosts.GetInstanceKeys() { discoveryInstanceKeys <- slaveKey } // Investigate master: discoveryInstanceKeys <- instance.MasterKey Cleanup: } // Start discovery begins a one time asynchronuous discovery process for the given // instance and all of its topology connected instances. // That is, the instance will be investigated for master and slaves, and the routines will follow on // each and every such found master/slave. // In essense, assuming all slaves in a replication topology are running, and given a single instance // in such topology, this function will detect the entire topology. func StartDiscovery(instanceKey inst.InstanceKey) { log.Infof("Starting discovery at %+v", instanceKey) pendingTokens := make(chan bool, maxConcurrency) completedTokens := make(chan bool, maxConcurrency) AccountedDiscoverInstance(instanceKey, pendingTokens, completedTokens) go handleDiscoveryRequests(pendingTokens, completedTokens) // Block until all are complete for { select { case <-pendingTokens: <-completedTokens default: inst.AuditOperation("start-discovery", &instanceKey, "") return } } } // ContinuousDiscovery starts an asynchronuous infinite discovery process where instances are // periodically investigated and their status captured, and long since unseen instances are // purged and forgotten. func ContinuousDiscovery() { log.Infof("Starting continuous discovery") go handleDiscoveryRequests(nil, nil) tick := time.Tick(time.Duration(config.Config.DiscoveryPollSeconds) * time.Second) forgetUnseenTick := time.Tick(time.Hour) for _ = range tick { instanceKeys, _ := inst.ReadOutdatedInstanceKeys() log.Debugf("outdated keys: %+v", instanceKeys) for _, instanceKey := range instanceKeys { discoveryInstanceKeys <- instanceKey } // See if we should also forget instances (lower frequency) select { case <-forgetUnseenTick: inst.ForgetLongUnseenInstances() default: } } }
func (s *TestSuite) TestSlavesAreSiblings(c *C) { i0, _ := inst.ReadTopologyInstance(&slave1Key) i1, _ := inst.ReadTopologyInstance(&slave2Key) c.Assert(inst.InstancesAreSiblings(i0, i1), Equals, true) }
func (s *TestSuite) TestNonSiblings(c *C) { i0, _ := inst.ReadTopologyInstance(&masterKey) i1, _ := inst.ReadTopologyInstance(&slave1Key) c.Assert(inst.InstancesAreSiblings(i0, i1), Not(Equals), true) }
func (s *TestSuite) TestInstanceIsMasterOf(c *C) { i0, _ := inst.ReadTopologyInstance(&masterKey) i1, _ := inst.ReadTopologyInstance(&slave1Key) c.Assert(inst.InstanceIsMasterOf(i0, i1), Equals, true) }