func TestNoExtrapolatedMySQLReplicationLag(t *testing.T) {
mysqld := mysqlctl.NewFakeMysqlDaemon(nil)
mysqld.Replicating = true
mysqld.SecondsBehindMaster = 10
slaveStopped := true
now := time.Now()
rep := &replicationReporter{
agent: &ActionAgent{MysqlDaemon: mysqld, _slaveStopped: &slaveStopped},
now: func() time.Time { return now },
}
// seed the last known value with a good value
dur, err := rep.Report(true, true)
if err != nil || dur != 10*time.Second {
t.Fatalf("wrong Report result: %v %v", dur, err)
}
// now 20 seconds later, mysqld is down
now = now.Add(20 * time.Second)
mysqld.SlaveStatusError = errors.New("mysql is down")
dur, err = rep.Report(true, true)
if err != mysqld.SlaveStatusError {
t.Fatalf("wrong Report error: %v", err)
}
}
func TestExtrapolatedMySQLReplicationLag(t *testing.T) {
mysqld := mysqlctl.NewFakeMysqlDaemon(nil)
mysqld.Replicating = true
mysqld.SecondsBehindMaster = 10
slaveStopped := true
now := time.Now()
rep := &replicationReporter{
agent: &ActionAgent{MysqlDaemon: mysqld, _slaveStopped: &slaveStopped},
now: func() time.Time { return now },
}
// seed the last known value with a good value
dur, err := rep.Report(true, true)
if err != nil || dur != 10*time.Second {
t.Fatalf("wrong Report result: %v %v", dur, err)
}
// now 20 seconds later, we're not replicating any more,
// we should get 20 more seconds in lag
now = now.Add(20 * time.Second)
mysqld.Replicating = false
dur, err = rep.Report(true, true)
if err != nil || dur != 30*time.Second {
t.Fatalf("wrong Report result: %v %v", dur, err)
}
}
func TestNoKnownMySQLReplicationLag(t *testing.T) {
mysqld := mysqlctl.NewFakeMysqlDaemon(nil)
mysqld.Replicating = false
slaveStopped := true
rep := &replicationReporter{
agent: &ActionAgent{MysqlDaemon: mysqld, _slaveStopped: &slaveStopped},
now: time.Now,
}
dur, err := rep.Report(true, true)
if err != health.ErrSlaveNotRunning {
t.Fatalf("wrong Report result: %v %v", dur, err)
}
}
func TestBasicMySQLReplicationLag(t *testing.T) {
mysqld := mysqlctl.NewFakeMysqlDaemon(nil)
mysqld.Replicating = true
mysqld.SecondsBehindMaster = 10
slaveStopped := true
rep := &replicationReporter{
agent: &ActionAgent{MysqlDaemon: mysqld, _slaveStopped: &slaveStopped},
now: time.Now,
}
dur, err := rep.Report(true, true)
if err != nil || dur != 10*time.Second {
t.Fatalf("wrong Report result: %v %v", dur, err)
}
}
// NewFakeTablet creates the test tablet in the topology. 'uid'
// has to be between 0 and 99. All the tablet info will be derived
// from that. Look at the implementation if you need values.
// Use TabletOption implementations if you need to change values at creation.
func NewFakeTablet(t *testing.T, wr *wrangler.Wrangler, cell string, uid uint32, tabletType topodatapb.TabletType, db *fakesqldb.DB, options ...TabletOption) *FakeTablet {
if uid < 0 || uid > 99 {
t.Fatalf("uid has to be between 0 and 99: %v", uid)
}
tablet := &topodatapb.Tablet{
Alias: &topodatapb.TabletAlias{Cell: cell, Uid: uid},
Hostname: fmt.Sprintf("%vhost", cell),
PortMap: map[string]int32{
"vt": int32(8100 + uid),
"mysql": int32(3300 + uid),
"grpc": int32(8200 + uid),
},
Ip: fmt.Sprintf("%v.0.0.1", 100+uid),
Keyspace: "test_keyspace",
Shard: "0",
Type: tabletType,
}
for _, option := range options {
option(tablet)
}
_, startHTTPServer := tablet.PortMap["start_http_server"]
delete(tablet.PortMap, "start_http_server")
_, force := tablet.PortMap["force_init"]
delete(tablet.PortMap, "force_init")
if err := wr.InitTablet(context.Background(), tablet, force, false, true, false); err != nil {
t.Fatalf("cannot create tablet %v: %v", uid, err)
}
// create a FakeMysqlDaemon with the right information by default
fakeMysqlDaemon := mysqlctl.NewFakeMysqlDaemon(db)
fakeMysqlDaemon.MysqlPort = 3300 + int32(uid)
return &FakeTablet{
Tablet: tablet,
FakeMysqlDaemon: fakeMysqlDaemon,
StartHTTPServer: startHTTPServer,
}
}
// NewFakeTablet creates the test tablet in the topology. 'uid'
// has to be between 0 and 99. All the tablet info will be derived
// from that. Look at the implementation if you need values.
// Use TabletOption implementations if you need to change values at creation.
func NewFakeTablet(t *testing.T, wr *wrangler.Wrangler, cell string, uid uint32, tabletType topo.TabletType, options ...TabletOption) *FakeTablet {
if uid < 0 || uid > 99 {
t.Fatalf("uid has to be between 0 and 99: %v", uid)
}
tablet := &topo.Tablet{
Alias: topo.TabletAlias{Cell: cell, Uid: uid},
Hostname: fmt.Sprintf("%vhost", cell),
Portmap: map[string]int{
"vt": 8100 + int(uid),
"mysql": 3300 + int(uid),
"vts": 8200 + int(uid),
},
IPAddr: fmt.Sprintf("%v.0.0.1", 100+uid),
Keyspace: "test_keyspace",
Shard: "0",
Type: tabletType,
}
for _, option := range options {
option(tablet)
}
delete(tablet.Portmap, "parent_uid")
_, force := tablet.Portmap["force_init"]
delete(tablet.Portmap, "force_init")
if err := wr.InitTablet(context.Background(), tablet, force, true, false); err != nil {
t.Fatalf("cannot create tablet %v: %v", uid, err)
}
// create a FakeMysqlDaemon with the right information by default
fakeMysqlDaemon := mysqlctl.NewFakeMysqlDaemon()
fakeMysqlDaemon.MysqlPort = 3300 + int(uid)
return &FakeTablet{
Tablet: tablet,
FakeMysqlDaemon: fakeMysqlDaemon,
}
}
// TestInitTablet will test the InitTablet code creates / updates the
// tablet node correctly. Note we modify global parameters (the flags)
// so this has to be in one test.
func TestInitTablet(t *testing.T) {
ctx := context.Background()
db := fakesqldb.Register()
ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
tabletAlias := &pb.TabletAlias{
Cell: "cell1",
Uid: 1,
}
// start with idle, and a tablet record that doesn't exist
port := int32(1234)
gRPCPort := int32(3456)
mysqlDaemon := mysqlctl.NewFakeMysqlDaemon(db)
agent := &ActionAgent{
TopoServer: ts,
TabletAlias: tabletAlias,
MysqlDaemon: mysqlDaemon,
DBConfigs: dbconfigs.DBConfigs{},
SchemaOverrides: nil,
BinlogPlayerMap: nil,
LockTimeout: 10 * time.Second,
batchCtx: ctx,
History: history.New(historyLength),
lastHealthMapCount: new(stats.Int),
_healthy: fmt.Errorf("healthcheck not run yet"),
}
*initTabletType = "idle"
*tabletHostname = "localhost"
if err := agent.InitTablet(port, gRPCPort); err != nil {
t.Fatalf("NewTestActionAgent(idle) failed: %v", err)
}
ti, err := ts.GetTablet(ctx, tabletAlias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
if ti.Type != pb.TabletType_IDLE {
t.Errorf("wrong type for tablet: %v", ti.Type)
}
if ti.Hostname != "localhost" {
t.Errorf("wrong hostname for tablet: %v", ti.Hostname)
}
if ti.PortMap["vt"] != port {
t.Errorf("wrong port for tablet: %v", ti.PortMap["vt"])
}
if ti.PortMap["grpc"] != gRPCPort {
t.Errorf("wrong gRPC port for tablet: %v", ti.PortMap["grpc"])
}
// try again now that the node exists
port = 3456
if err := agent.InitTablet(port, gRPCPort); err != nil {
t.Fatalf("NewTestActionAgent(idle again) failed: %v", err)
}
ti, err = ts.GetTablet(ctx, tabletAlias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
if ti.PortMap["vt"] != port {
t.Errorf("wrong port for tablet: %v", ti.PortMap["vt"])
}
if ti.PortMap["grpc"] != gRPCPort {
t.Errorf("wrong gRPC port for tablet: %v", ti.PortMap["grpc"])
}
// try with a keyspace and shard on the previously idle tablet,
// should fail
*initTabletType = "replica"
*initKeyspace = "test_keyspace"
*initShard = "-80"
if err := agent.InitTablet(port, gRPCPort); err == nil || !strings.Contains(err.Error(), "InitTablet failed because existing tablet keyspace and shard / differ from the provided ones test_keyspace/-80") {
t.Fatalf("InitTablet(type over idle) didn't fail correctly: %v", err)
}
// now let's use a different real tablet in a shard, that will create
// the keyspace and shard.
tabletAlias = &pb.TabletAlias{
Cell: "cell1",
Uid: 2,
}
agent.TabletAlias = tabletAlias
if err := agent.InitTablet(port, gRPCPort); err != nil {
t.Fatalf("InitTablet(type) failed: %v", err)
}
si, err := ts.GetShard(ctx, "test_keyspace", "-80")
if err != nil {
t.Fatalf("GetShard failed: %v", err)
}
if len(si.Cells) != 1 || si.Cells[0] != "cell1" {
t.Errorf("shard.Cells not updated properly: %v", si)
}
ti, err = ts.GetTablet(ctx, tabletAlias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
if ti.Type != pb.TabletType_REPLICA {
t.Errorf("wrong tablet type: %v", ti.Type)
}
// try to init again, this time with health check on
*initTabletType = ""
*targetTabletType = "replica"
if err := agent.InitTablet(port, gRPCPort); err != nil {
t.Fatalf("InitTablet(type, healthcheck) failed: %v", err)
}
ti, err = ts.GetTablet(ctx, tabletAlias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
if ti.Type != pb.TabletType_SPARE {
t.Errorf("wrong tablet type: %v", ti.Type)
}
// update shard's master to our alias, then try to init again
si, err = ts.GetShard(ctx, "test_keyspace", "-80")
if err != nil {
t.Fatalf("GetShard failed: %v", err)
}
si.MasterAlias = tabletAlias
if err := ts.UpdateShard(ctx, si); err != nil {
t.Fatalf("UpdateShard failed: %v", err)
}
if err := agent.InitTablet(port, gRPCPort); err != nil {
t.Fatalf("InitTablet(type, healthcheck) failed: %v", err)
}
ti, err = ts.GetTablet(ctx, tabletAlias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
if ti.Type != pb.TabletType_MASTER {
t.Errorf("wrong tablet type: %v", ti.Type)
}
// init again with the tablet_type set, no healthcheck
// (also check db name override and tags here)
*initTabletType = "replica"
*targetTabletType = ""
*initDbNameOverride = "DBNAME"
initTags.Set("aaa:bbb")
if err := agent.InitTablet(port, gRPCPort); err != nil {
t.Fatalf("InitTablet(type, healthcheck) failed: %v", err)
}
ti, err = ts.GetTablet(ctx, tabletAlias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
if ti.Type != pb.TabletType_MASTER {
t.Errorf("wrong tablet type: %v", ti.Type)
}
if ti.DbNameOverride != "DBNAME" {
t.Errorf("wrong tablet DbNameOverride: %v", ti.DbNameOverride)
}
if len(ti.Tags) != 1 || ti.Tags["aaa"] != "bbb" {
t.Errorf("wrong tablet tags: %v", ti.Tags)
}
}
// TestInitTablet will test the InitTablet code creates / updates the
// tablet node correctly. Note we modify global parameters (the flags)
// so this has to be in one test.
func TestInitTablet(t *testing.T) {
ctx := context.Background()
db := fakesqldb.Register()
ts := zktestserver.New(t, []string{"cell1", "cell2"})
tabletAlias := &topodatapb.TabletAlias{
Cell: "cell1",
Uid: 1,
}
// start with a tablet record that doesn't exist
port := int32(1234)
gRPCPort := int32(3456)
mysqlDaemon := mysqlctl.NewFakeMysqlDaemon(db)
agent := &ActionAgent{
TopoServer: ts,
TabletAlias: tabletAlias,
MysqlDaemon: mysqlDaemon,
DBConfigs: dbconfigs.DBConfigs{},
SchemaOverrides: nil,
BinlogPlayerMap: nil,
batchCtx: ctx,
History: history.New(historyLength),
lastHealthMapCount: new(stats.Int),
_healthy: fmt.Errorf("healthcheck not run yet"),
}
// let's use a real tablet in a shard, that will create
// the keyspace and shard.
*tabletHostname = "localhost"
*initTabletType = "replica"
*initKeyspace = "test_keyspace"
*initShard = "-80"
tabletAlias = &topodatapb.TabletAlias{
Cell: "cell1",
Uid: 2,
}
agent.TabletAlias = tabletAlias
if err := agent.InitTablet(port, gRPCPort); err != nil {
t.Fatalf("InitTablet(type) failed: %v", err)
}
si, err := ts.GetShard(ctx, "test_keyspace", "-80")
if err != nil {
t.Fatalf("GetShard failed: %v", err)
}
if len(si.Cells) != 1 || si.Cells[0] != "cell1" {
t.Errorf("shard.Cells not updated properly: %v", si)
}
ti, err := ts.GetTablet(ctx, tabletAlias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
if ti.Type != topodatapb.TabletType_REPLICA {
t.Errorf("wrong tablet type: %v", ti.Type)
}
if ti.Hostname != "localhost" {
t.Errorf("wrong hostname for tablet: %v", ti.Hostname)
}
if ti.PortMap["vt"] != port {
t.Errorf("wrong port for tablet: %v", ti.PortMap["vt"])
}
if ti.PortMap["grpc"] != gRPCPort {
t.Errorf("wrong gRPC port for tablet: %v", ti.PortMap["grpc"])
}
// try to init again, this time with health check on
*initTabletType = ""
*targetTabletType = "replica"
if err := agent.InitTablet(port, gRPCPort); err != nil {
t.Fatalf("InitTablet(type, healthcheck) failed: %v", err)
}
ti, err = ts.GetTablet(ctx, tabletAlias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
if ti.Type != topodatapb.TabletType_SPARE {
t.Errorf("wrong tablet type: %v", ti.Type)
}
// update shard's master to our alias, then try to init again
si, err = ts.GetShard(ctx, "test_keyspace", "-80")
if err != nil {
t.Fatalf("GetShard failed: %v", err)
}
si.MasterAlias = tabletAlias
if err := ts.UpdateShard(ctx, si); err != nil {
t.Fatalf("UpdateShard failed: %v", err)
}
if err := agent.InitTablet(port, gRPCPort); err != nil {
t.Fatalf("InitTablet(type, healthcheck) failed: %v", err)
}
ti, err = ts.GetTablet(ctx, tabletAlias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
// It should still be spare, because the tablet record doesn't agree.
if ti.Type != topodatapb.TabletType_SPARE {
t.Errorf("wrong tablet type: %v", ti.Type)
}
// Fix the tablet record to agree that we're master.
ti.Type = topodatapb.TabletType_MASTER
if err := ts.UpdateTablet(ctx, ti); err != nil {
t.Fatalf("UpdateTablet failed: %v", err)
}
if err := agent.InitTablet(port, gRPCPort); err != nil {
t.Fatalf("InitTablet(type, healthcheck) failed: %v", err)
}
ti, err = ts.GetTablet(ctx, tabletAlias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
if ti.Type != topodatapb.TabletType_MASTER {
t.Errorf("wrong tablet type: %v", ti.Type)
}
// init again with the tablet_type set, no healthcheck
// (also check db name override and tags here)
*initTabletType = "replica"
*targetTabletType = ""
*initDbNameOverride = "DBNAME"
initTags.Set("aaa:bbb")
if err := agent.InitTablet(port, gRPCPort); err != nil {
t.Fatalf("InitTablet(type, healthcheck) failed: %v", err)
}
ti, err = ts.GetTablet(ctx, tabletAlias)
if err != nil {
t.Fatalf("GetTablet failed: %v", err)
}
if ti.Type != topodatapb.TabletType_MASTER {
t.Errorf("wrong tablet type: %v", ti.Type)
}
if ti.DbNameOverride != "DBNAME" {
t.Errorf("wrong tablet DbNameOverride: %v", ti.DbNameOverride)
}
if len(ti.Tags) != 1 || ti.Tags["aaa"] != "bbb" {
t.Errorf("wrong tablet tags: %v", ti.Tags)
}
}