func (s *TestSuite) TestBeginMaintenance(c *C) {
clearTestMaintenance()
_, _ = inst.ReadTopologyInstance(&masterKey)
_, err := inst.BeginMaintenance(&masterKey, "unittest", "TestBeginMaintenance")
c.Assert(err, IsNil)
}
func (s *TestSuite) TestReadTopologyAndInstanceSlave(c *C) {
i, _ := inst.ReadTopologyInstance(&slave1Key)
iRead, found, _ := inst.ReadInstance(&slave1Key)
c.Assert(found, Equals, true)
c.Assert(iRead.Key.Hostname, Equals, i.Key.Hostname)
c.Assert(iRead.Version, Equals, i.Version)
}
func (s *TestSuite) TestReadTopologySlave(c *C) {
key := slave1Key
i, _ := inst.ReadTopologyInstance(&key)
c.Assert(i.Key.Hostname, Equals, key.Hostname)
c.Assert(i.IsSlave(), Equals, true)
c.Assert(len(i.SlaveHosts), Equals, 0)
}
func (s *TestSuite) TestForgetMaster(c *C) {
_, _ = inst.ReadTopologyInstance(&masterKey)
_, found, _ := inst.ReadInstance(&masterKey)
c.Assert(found, Equals, true)
inst.ForgetInstance(&masterKey)
_, found, _ = inst.ReadInstance(&masterKey)
c.Assert(found, Equals, false)
}
func (s *TestSuite) TestGetMasterOfASlave(c *C) {
i, err := inst.ReadTopologyInstance(&slave1Key)
c.Assert(err, IsNil)
master, err := inst.GetInstanceMaster(i)
c.Assert(err, IsNil)
c.Assert(master.IsSlave(), Equals, false)
c.Assert(master.Key.Port, Equals, 22987)
}
func (s *TestSuite) TestReadTopologyAndInstanceMaster(c *C) {
i, _ := inst.ReadTopologyInstance(&masterKey)
iRead, found, _ := inst.ReadInstance(&masterKey)
c.Assert(found, Equals, true)
c.Assert(iRead.Key.Hostname, Equals, i.Key.Hostname)
c.Assert(iRead.Version, Equals, i.Version)
c.Assert(len(iRead.SlaveHosts), Equals, len(i.SlaveHosts))
}
func (s *TestSuite) TestReadTopologyUnexisting(c *C) {
key := inst.InstanceKey{
Hostname: "127.0.0.1",
Port: 22999,
}
_, err := inst.ReadTopologyInstance(&key)
c.Assert(err, Not(IsNil))
}
func (s *TestSuite) TestMakeCoMasterAndBack(c *C) {
clearTestMaintenance()
slave1, err := inst.MakeCoMaster(&slave1Key)
c.Assert(err, IsNil)
// Now master & slave1 expected to be co-masters. Check!
master, _ := inst.ReadTopologyInstance(&masterKey)
c.Assert(master.MasterKey.Port, Not(Equals), inst.InvalidPort)
c.Assert(master.IsSlaveOf(slave1), Equals, true)
c.Assert(slave1.IsSlaveOf(master), Equals, true)
// detach - resotre to original state
master, err = inst.DetachSlaveFromMaster(&masterKey)
slave1, _ = inst.ReadTopologyInstance(&slave1Key)
c.Assert(err, IsNil)
c.Assert(master.MasterKey.Port, Equals, inst.InvalidPort)
}
func (s *TestSuite) TestReadTopologyMaster(c *C) {
key := masterKey
i, _ := inst.ReadTopologyInstance(&key)
c.Assert(i.Key.Hostname, Equals, key.Hostname)
c.Assert(i.IsSlave(), Equals, false)
c.Assert(len(i.SlaveHosts), Equals, 3)
c.Assert(len(i.SlaveHosts.GetInstanceKeys()), Equals, len(i.SlaveHosts))
}
func (s *TestSuite) TestFailEndMaintenanceTwice(c *C) {
clearTestMaintenance()
_, _ = inst.ReadTopologyInstance(&masterKey)
k, err := inst.BeginMaintenance(&masterKey, "unittest", "TestFailEndMaintenanceTwice")
c.Assert(err, IsNil)
err = inst.EndMaintenance(k)
c.Assert(err, IsNil)
err = inst.EndMaintenance(k)
c.Assert(err, Not(IsNil))
}
func (s *TestSuite) TestFailMoveBelowUponMaintenance(c *C) {
clearTestMaintenance()
_, _ = inst.ReadTopologyInstance(&slave1Key)
k, err := inst.BeginMaintenance(&slave1Key, "unittest", "TestBeginEndMaintenance")
c.Assert(err, IsNil)
_, err = inst.MoveBelow(&slave1Key, &slave2Key)
c.Assert(err, Not(IsNil))
err = inst.EndMaintenance(k)
c.Assert(err, IsNil)
}
func (s *TestSuite) TestStopStartSlave(c *C) {
i, _ := inst.ReadTopologyInstance(&slave1Key)
c.Assert(i.SlaveRunning(), Equals, true)
i, _ = inst.StopSlaveNicely(&i.Key)
c.Assert(i.SlaveRunning(), Equals, false)
c.Assert(i.SQLThreadUpToDate(), Equals, true)
i, _ = inst.StartSlave(&i.Key)
c.Assert(i.SlaveRunning(), Equals, true)
}
func (s *TestSuite) TestFailMoveBelowUponOtherSlaveStopped(c *C) {
clearTestMaintenance()
slave1, _ := inst.ReadTopologyInstance(&slave1Key)
c.Assert(slave1.SlaveRunning(), Equals, true)
slave1, _ = inst.StopSlaveNicely(&slave1.Key)
c.Assert(slave1.SlaveRunning(), Equals, false)
_, err := inst.MoveBelow(&slave2Key, &slave1Key)
c.Assert(err, Not(IsNil))
_, _ = inst.StartSlave(&slave1.Key)
}
func (s *TestSuite) TestMoveBelowAndBackComplex(c *C) {
clearTestMaintenance()
// become child
slave1, _ := inst.MoveBelow(&slave1Key, &slave2Key)
c.Assert(slave1.MasterKey.Equals(&slave2Key), Equals, true)
c.Assert(slave1.SlaveRunning(), Equals, true)
// Now let's have fun. Stop slave2 (which is now parent of slave1), execute queries on master,
// move s1 back under master, start all, verify queries.
_, err := inst.StopSlave(&slave2Key)
c.Assert(err, IsNil)
randValue := rand.Int()
_, err = inst.ExecInstance(&masterKey, `replace into orchestrator_test.test_table (name, value) values ('TestMoveBelowAndBackComplex', ?)`, randValue)
c.Assert(err, IsNil)
master, err := inst.ReadTopologyInstance(&masterKey)
c.Assert(err, IsNil)
// And back; keep topology intact
slave1, err = inst.MoveUp(&slave1Key)
c.Assert(err, IsNil)
_, err = inst.MasterPosWait(&slave1Key, &master.SelfBinlogCoordinates)
c.Assert(err, IsNil)
slave2, err := inst.ReadTopologyInstance(&slave2Key)
c.Assert(err, IsNil)
_, err = inst.MasterPosWait(&slave2Key, &master.SelfBinlogCoordinates)
c.Assert(err, IsNil)
// Now check for value!
var value1, value2 int
inst.ScanInstanceRow(&slave1Key, `select value from orchestrator_test.test_table where name='TestMoveBelowAndBackComplex'`, &value1)
inst.ScanInstanceRow(&slave2Key, `select value from orchestrator_test.test_table where name='TestMoveBelowAndBackComplex'`, &value2)
c.Assert(inst.InstancesAreSiblings(slave1, slave2), Equals, true)
c.Assert(value1, Equals, randValue)
c.Assert(value2, Equals, randValue)
}
func (s *TestSuite) TestDiscover(c *C) {
var err error
_, err = db.ExecOrchestrator("delete from database_instance where hostname = ? and port = ?", masterKey.Hostname, masterKey.Port)
_, err = db.ExecOrchestrator("delete from database_instance where hostname = ? and port = ?", slave1Key.Hostname, slave1Key.Port)
_, err = db.ExecOrchestrator("delete from database_instance where hostname = ? and port = ?", slave2Key.Hostname, slave2Key.Port)
_, err = db.ExecOrchestrator("delete from database_instance where hostname = ? and port = ?", slave3Key.Hostname, slave3Key.Port)
_, found, _ := inst.ReadInstance(&masterKey)
c.Assert(found, Equals, false)
_, _ = inst.ReadTopologyInstance(&slave1Key)
orchestrator.StartDiscovery(slave1Key)
_, found, err = inst.ReadInstance(&slave1Key)
c.Assert(found, Equals, true)
c.Assert(err, IsNil)
}
func (s *TestSuite) TestMoveBelowAndBack(c *C) {
clearTestMaintenance()
// become child
slave1, err := inst.MoveBelow(&slave1Key, &slave2Key)
c.Assert(err, IsNil)
c.Assert(slave1.MasterKey.Equals(&slave2Key), Equals, true)
c.Assert(slave1.SlaveRunning(), Equals, true)
// And back; keep topology intact
slave1, _ = inst.MoveUp(&slave1Key)
slave2, _ := inst.ReadTopologyInstance(&slave2Key)
c.Assert(inst.InstancesAreSiblings(slave1, slave2), Equals, true)
c.Assert(slave1.SlaveRunning(), Equals, true)
}
// DiscoverInstance will attempt discovering an instance (unless it is already up to date) and will
// list down its master and slaves (if any) for further discovery.
func DiscoverInstance(instanceKey inst.InstanceKey) {
instanceKey.Formalize()
if !instanceKey.IsValid() {
return
}
instance, found, err := inst.ReadInstance(&instanceKey)
if found && instance.IsUpToDate && instance.IsLastCheckValid {
// we've already discovered this one. Skip!
goto Cleanup
}
// First we've ever heard of this instance. Continue investigation:
instance, err = inst.ReadTopologyInstance(&instanceKey)
// panic can occur (IO stuff). Therefore it may happen
// that instance is nil. Check it.
if err != nil || instance == nil {
goto Cleanup
}
fmt.Printf("host: %+v, master: %+v\n", instance.Key, instance.MasterKey)
// Investigate slaves:
for _, slaveKey := range instance.SlaveHosts.GetInstanceKeys() {
discoveryInstanceKeys <- slaveKey
}
// Investigate master:
discoveryInstanceKeys <- instance.MasterKey
Cleanup:
}
// Start discovery begins a one time asynchronuous discovery process for the given
// instance and all of its topology connected instances.
// That is, the instance will be investigated for master and slaves, and the routines will follow on
// each and every such found master/slave.
// In essense, assuming all slaves in a replication topology are running, and given a single instance
// in such topology, this function will detect the entire topology.
func StartDiscovery(instanceKey inst.InstanceKey) {
log.Infof("Starting discovery at %+v", instanceKey)
pendingTokens := make(chan bool, maxConcurrency)
completedTokens := make(chan bool, maxConcurrency)
AccountedDiscoverInstance(instanceKey, pendingTokens, completedTokens)
go handleDiscoveryRequests(pendingTokens, completedTokens)
// Block until all are complete
for {
select {
case <-pendingTokens:
<-completedTokens
default:
inst.AuditOperation("start-discovery", &instanceKey, "")
return
}
}
}
// ContinuousDiscovery starts an asynchronuous infinite discovery process where instances are
// periodically investigated and their status captured, and long since unseen instances are
// purged and forgotten.
func ContinuousDiscovery() {
log.Infof("Starting continuous discovery")
go handleDiscoveryRequests(nil, nil)
tick := time.Tick(time.Duration(config.Config.DiscoveryPollSeconds) * time.Second)
forgetUnseenTick := time.Tick(time.Hour)
for _ = range tick {
instanceKeys, _ := inst.ReadOutdatedInstanceKeys()
log.Debugf("outdated keys: %+v", instanceKeys)
for _, instanceKey := range instanceKeys {
discoveryInstanceKeys <- instanceKey
}
// See if we should also forget instances (lower frequency)
select {
case <-forgetUnseenTick:
inst.ForgetLongUnseenInstances()
default:
}
}
}
func (s *TestSuite) TestSlavesAreSiblings(c *C) {
i0, _ := inst.ReadTopologyInstance(&slave1Key)
i1, _ := inst.ReadTopologyInstance(&slave2Key)
c.Assert(inst.InstancesAreSiblings(i0, i1), Equals, true)
}
func (s *TestSuite) TestNonSiblings(c *C) {
i0, _ := inst.ReadTopologyInstance(&masterKey)
i1, _ := inst.ReadTopologyInstance(&slave1Key)
c.Assert(inst.InstancesAreSiblings(i0, i1), Not(Equals), true)
}
func (s *TestSuite) TestInstanceIsMasterOf(c *C) {
i0, _ := inst.ReadTopologyInstance(&masterKey)
i1, _ := inst.ReadTopologyInstance(&slave1Key)
c.Assert(inst.InstanceIsMasterOf(i0, i1), Equals, true)
}