func TestStream(t *testing.T) {
env := setup("cat $3", 0)
defer cleanup(env)
var transactions []transaction
out, err := ioutil.ReadFile(testfiles.Locate("mysqlctl_test/expected.json"))
if err != nil {
t.Fatal(err)
}
err = json.Unmarshal(out, &transactions)
if err != nil {
t.Fatal(err)
}
svm := &sync2.ServiceManager{}
curTransaction := 0
sendTx := func(tx *proto.BinlogTransaction) error {
for i, stmt := range tx.Statements {
if transactions[curTransaction].Statements[i].Sql != string(stmt.Sql) {
t.Errorf("want %s, got %s", transactions[curTransaction].Statements[i].Sql, stmt.Sql)
}
if transactions[curTransaction].Statements[i].Category != stmt.Category {
t.Errorf("want %d, got %d", transactions[curTransaction].Statements[i].Category, stmt.Category)
}
}
if transactions[curTransaction].GTIDField != tx.GTIDField {
t.Errorf("want %#v, got %#v", transactions[curTransaction].GTIDField, tx.GTIDField)
}
curTransaction++
if curTransaction == len(transactions) {
// Launch as goroutine to prevent deadlock.
go svm.Stop()
}
// Uncomment the following lines to produce a different set of
// expected outputs. You'll need to massage the file a bit afterwards.
/*
fmt.Printf("{\n\"Statements\": [\n")
for i := 0; i < len(tx.Statements); i++ {
fmt.Printf(`{"Category": %d, "Sql": %#v}`, tx.Statements[i].Category, string(tx.Statements[i].Sql))
if i == len(tx.Statements)-1 {
fmt.Printf("\n")
} else {
fmt.Printf(",\n")
}
}
fmt.Printf("],\n")
fmt.Printf("\"GTID\": \"%s\"\n},\n", tx.GTID)
*/
return nil
}
bls := newTestBinlogFileStreamer("db", testfiles.Locate("mysqlctl_test/vt-0000041983-bin"), myproto.ReplicationPosition{}, sendTx)
svm.Go(func(ctx *sync2.ServiceContext) error {
return bls.streamFilePos(ctx, "vt-0000041983-bin.000001", 0)
})
err = svm.Join()
if err != nil {
t.Error(err)
}
}
func TestStream(t *testing.T) {
env := setup("cat $3", 0)
defer cleanup(env)
var transactions []transaction
out, err := ioutil.ReadFile(testfiles.Locate("mysqlctl_test/expected.json"))
if err != nil {
t.Fatal(err)
}
err = json.Unmarshal(out, &transactions)
if err != nil {
t.Fatal(err)
}
curTransaction := 0
bls := NewBinlogStreamer("db", testfiles.Locate("mysqlctl_test/vt-0000041983-bin"))
err = bls.Stream("vt-0000041983-bin.000001", 0, func(tx *proto.BinlogTransaction) error {
for i, stmt := range tx.Statements {
if transactions[curTransaction].Statements[i].Sql != string(stmt.Sql) {
t.Errorf("want %s, got %s", transactions[curTransaction].Statements[i].Sql, stmt.Sql)
}
if transactions[curTransaction].Statements[i].Category != stmt.Category {
t.Errorf("want %d, got %d", transactions[curTransaction].Statements[i].Category, stmt.Category)
}
}
if transactions[curTransaction].GroupId != tx.GroupId {
t.Errorf("want %#v, got %#v", transactions[curTransaction].GroupId, tx.GroupId)
}
curTransaction++
if curTransaction == len(transactions) {
// Launch as goroutine to prevent deadlock.
go bls.Stop()
}
// Uncomment the following lines to produce a different set of
// expected outputs. You'll need to massage the file a bit afterwards.
/*
fmt.Printf("{\n\"Statements\": [\n")
for i := 0; i < len(tx.Statements); i++ {
fmt.Printf(`{"Category": %d, "Sql": %#v}`, tx.Statements[i].Category, string(tx.Statements[i].Sql))
if i == len(tx.Statements)-1 {
fmt.Printf("\n")
} else {
fmt.Printf(",\n")
}
}
fmt.Printf("],\n")
fmt.Printf("\"GroupId\": \"%s\"\n},\n", tx.GroupId)
*/
return nil
})
if err != nil {
t.Error(err)
}
}
func TestFromFile(t *testing.T) {
conn := NewConnFromFile(testfiles.Locate("fakezk_test_config.json"))
keyspaces, _, err := conn.Children("/zk/testing/vt/ns")
if err != nil {
t.Errorf("conn.Children: %v", err)
}
if len(keyspaces) != 1 || keyspaces[0] != "test_keyspace" {
t.Errorf("conn.Children returned bad value: %v", keyspaces)
}
data, _, err := conn.Get("/zk/testing/vt/ns/test_keyspace")
if err != nil {
t.Errorf("conn.Get(/zk/testing/vt/ns/test_keyspace): %v", err)
}
if !strings.Contains(string(data), "ShardReferences") {
t.Errorf("conn.Get(/zk/testing/vt/ns/test_keyspace) returned bad value: %v", data)
}
data, _, err = conn.Get("/zk/testing/vt/ns/test_keyspace/0/master")
if err != nil {
t.Errorf("conn.Get(/zk/testing/vt/ns/test_keyspace/0/master): %v", err)
}
if !strings.Contains(string(data), "NamedPortMap") {
t.Errorf("conn.Get(/zk/testing/vt/ns/test_keyspace/0/master) returned bad value: %v", data)
}
}
func iterateFile(name string) (testCaseIterator chan testCase) {
name = testfiles.Locate("sqlparser_test/" + name)
fd, err := os.OpenFile(name, os.O_RDONLY, 0)
if err != nil {
panic(fmt.Sprintf("Could not open file %s", name))
}
testCaseIterator = make(chan testCase)
go func() {
defer close(testCaseIterator)
r := bufio.NewReader(fd)
lineno := 0
for {
line, err := r.ReadString('\n')
lines := strings.Split(strings.TrimRight(line, "\n"), "#")
lineno++
if err != nil {
if err != io.EOF {
panic(fmt.Sprintf("Error reading file %s: %s", name, err.Error()))
}
break
}
input := lines[0]
output := ""
if len(lines) > 1 {
output = lines[1]
}
if input == "" {
continue
}
testCaseIterator <- testCase{lineno, input, output}
}
}()
return testCaseIterator
}
func newSpecialReader(t *testing.T, filename string) *specialReader {
filename = testfiles.Locate(filename)
b, err := ioutil.ReadFile(filename)
if err != nil {
t.Fatalf("Cannot read file %v: %v", filename, err)
}
return &specialReader{t, b, 0}
}
func TestInitWithInvalidConfigFile(t *testing.T) {
setUpTableACL(&simpleacl.Factory{})
defer func() {
err := recover()
if err == nil {
t.Fatalf("init should fail for an invalid config file")
}
}()
Init(testfiles.Locate("tableacl/invalid_tableacl_config.json"))
}
func iterateJSONFile(name string) (testCaseIterator chan testCase) {
name = testfiles.Locate("sqlparser_test/" + name)
fd, err := os.OpenFile(name, os.O_RDONLY, 0)
if err != nil {
panic(fmt.Sprintf("Could not open file %s", name))
}
testCaseIterator = make(chan testCase)
go func() {
defer close(testCaseIterator)
r := bufio.NewReader(fd)
lineno := 0
for {
binput, _, err := r.ReadLine()
input := string(binput)
lineno++
if err != nil {
if err != io.EOF {
panic(fmt.Sprintf("Error reading file %s: %s", name, err.Error()))
}
break
}
if input == "" || input[0] == '#' {
//fmt.Printf("%s\n", input)
continue
}
var output []byte
for {
l, err := r.ReadBytes('\n')
lineno++
if err != nil {
panic(fmt.Sprintf("Error reading file %s: %s", name, err.Error()))
}
output = append(output, l...)
if l[0] == '}' {
output = output[:len(output)-1]
b := bytes.NewBuffer(make([]byte, 0, 64))
if err := json.Compact(b, output); err == nil {
output = b.Bytes()
}
break
}
if l[0] == '"' {
output = output[1 : len(output)-2]
break
}
}
testCaseIterator <- testCase{lineno, input, string(output)}
}
}()
return testCaseIterator
}
// TestRoration should not hang
func TestRotation(t *testing.T) {
env := setup("cat $3", 0)
defer cleanup(env)
bls := NewBinlogStreamer("db", testfiles.Locate("mysqlctl_test/vt-0000041983-bin"))
err := bls.Stream("vt-0000041983-bin.000004", 2682, func(tx *proto.BinlogTransaction) error {
bls.Stop()
return nil
})
if err != nil {
t.Error(err)
}
}
func TestSimple(t *testing.T) {
input := testfiles.Locate("bson_test/simple_type.go")
b, err := ioutil.ReadFile(input)
if err != nil {
log.Fatal(err)
}
out, err := generateCode(string(b), "MyType")
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
return
}
fmt.Printf("%s\n", out)
}
// TestRoration should not hang
func TestRotation(t *testing.T) {
env := setup("cat $3", 0)
defer cleanup(env)
bls := newTestBinlogFileStreamer("db", testfiles.Locate("mysqlctl_test/vt-0000041983-bin"))
err := bls.streamFilePos("vt-0000041983-bin.000004", 2682, func(tx *proto.BinlogTransaction) error {
// Launch as goroutine to prevent deadlock.
go bls.Stop()
return nil
})
if err != nil {
t.Error(err)
}
}
// TestRoration should not hang
func TestRotation(t *testing.T) {
env := setup("cat $3", 0)
defer cleanup(env)
svm := &sync2.ServiceManager{}
bls := newTestBinlogFileStreamer("db", testfiles.Locate("mysqlctl_test/vt-0000041983-bin"), myproto.ReplicationPosition{}, func(tx *proto.BinlogTransaction) error {
// Launch as goroutine to prevent deadlock.
go svm.Stop()
return nil
})
svm.Go(func(ctx *sync2.ServiceContext) error {
return bls.streamFilePos(ctx, "vt-0000041983-bin.000004", 2682)
})
err := svm.Join()
if err != nil {
t.Error(err)
}
}
func main() {
input := testfiles.Locate("bson_test/simple_type.go")
b, err := ioutil.ReadFile(input)
if err != nil {
log.Fatal(err)
}
src := string(b)
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, "", src, 0)
if err != nil {
log.Fatal(err)
}
//ast.Print(fset, f)
typeInfo, err := FindType(f, "MyType")
if err != nil {
fmt.Println(err)
return
}
generator.ExecuteTemplate(os.Stdout, "Body", typeInfo)
}
func BenchmarkFileStreamerParseEvents(b *testing.B) {
filename := testfiles.Locate("binlog_test/vt-0000062347-bin.000001")
var svm sync2.ServiceManager
count := 0
bls := newTestBinlogFileStreamer("vt_test_database", "", myproto.ReplicationPosition{}, func(tx *proto.BinlogTransaction) error {
count++
return nil
})
for i := 0; i < b.N; i++ {
if err := bls.file.Init(filename, 0); err != nil {
b.Fatalf("%v", err)
}
svm.Go(bls.run)
if err := svm.Join(); err != nil {
b.Errorf("%v", err)
}
bls.file.Close()
}
b.Logf("%d transactions processed", count)
}
func BenchmarkConnStreamerParseEvents(b *testing.B) {
filename := testfiles.Locate("binlog_test/vt-0000062347-bin.000001")
var svm sync2.ServiceManager
count := 0
bls := &binlogConnStreamer{dbname: "vt_test_database", sendTransaction: func(tx *proto.BinlogTransaction) error {
count++
return nil
}}
for i := 0; i < b.N; i++ {
events := readEvents(b, filename)
svm.Go(func(svc *sync2.ServiceContext) error {
_, err := bls.parseEvents(svc, events)
return err
})
if err := svm.Join(); err != ServerEOF {
b.Errorf("%v", err)
}
}
b.Logf("%d transactions processed", count)
}
func readLines(t *testing.T, name string) []string {
file, err := os.Open(testfiles.Locate(name))
if err != nil {
t.Fatalf("Cannot open %v: %v", name, err)
}
r := NewCSVReader(file, ',')
lines := make([]string, 0)
for {
line, err := r.ReadRecord()
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
lines = append(lines, string(line))
}
return lines
}
func readData(t *testing.T, name string, numberColumn bool) []pair {
file, err := os.Open(testfiles.Locate(name))
if err != nil {
t.Fatalf("Cannot open %v: %v", name, err)
}
r := NewKeyspaceCSVReader(file, ',', numberColumn)
keyspaceIds := make([]pair, 0)
for {
kid, line, err := r.ReadRecord()
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
keyspaceIds = append(keyspaceIds, pair{kid, string(line)})
}
return keyspaceIds
}
func locateFile(name string) string {
if path.IsAbs(name) {
return name
}
return testfiles.Locate("vtgate/" + name)
}
func locateFile(name string) string {
if path.IsAbs(name) {
return name
}
return testfiles.Locate("sqlparser_test/" + name)
}
func createSetup(ctx context.Context, t *testing.T) (topo.Impl, topo.Impl) {
fromConn := fakezk.NewConn()
fromTS := zktopo.NewServer(fromConn)
toConn := fakezk.NewConn()
toTS := zktopo.NewServer(toConn)
for _, zkPath := range []string{"/zk/test_cell/vt", "/zk/global/vt"} {
if _, err := zk.CreateRecursive(fromConn, zkPath, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil {
t.Fatalf("cannot init fromTS: %v", err)
}
}
// create a keyspace and a couple tablets
if err := fromTS.CreateKeyspace(ctx, "test_keyspace", &topodatapb.Keyspace{}); err != nil {
t.Fatalf("cannot create keyspace: %v", err)
}
if err := fromTS.CreateShard(ctx, "test_keyspace", "0", &topodatapb.Shard{Cells: []string{"test_cell"}}); err != nil {
t.Fatalf("cannot create shard: %v", err)
}
tts := topo.Server{Impl: fromTS}
if err := tts.CreateTablet(ctx, &topodatapb.Tablet{
Alias: &topodatapb.TabletAlias{
Cell: "test_cell",
Uid: 123,
},
Hostname: "masterhost",
Ip: "1.2.3.4",
PortMap: map[string]int32{
"vt": 8101,
"gprc": 8102,
"mysql": 3306,
},
Keyspace: "test_keyspace",
Shard: "0",
Type: topodatapb.TabletType_MASTER,
DbNameOverride: "",
KeyRange: nil,
}); err != nil {
t.Fatalf("cannot create master tablet: %v", err)
}
if err := tts.CreateTablet(ctx, &topodatapb.Tablet{
Alias: &topodatapb.TabletAlias{
Cell: "test_cell",
Uid: 234,
},
Ip: "2.3.4.5",
PortMap: map[string]int32{
"vt": 8101,
"grpc": 8102,
"mysql": 3306,
},
Hostname: "slavehost",
Keyspace: "test_keyspace",
Shard: "0",
Type: topodatapb.TabletType_REPLICA,
DbNameOverride: "",
KeyRange: nil,
}); err != nil {
t.Fatalf("cannot create slave tablet: %v", err)
}
os.Setenv("ZK_CLIENT_CONFIG", testfiles.Locate("topo_helpers_test_zk_client.json"))
cells, err := fromTS.GetKnownCells(ctx)
if err != nil {
t.Fatalf("fromTS.GetKnownCells: %v", err)
}
log.Infof("Cells: %v", cells)
return fromTS, toTS
}
func locateFile(name string) string {
if path.IsAbs(name) {
return name
}
return testfiles.Locate("tabletserver/" + name)
}
func TestInitWithValidConfig(t *testing.T) {
setUpTableACL(&simpleacl.Factory{})
Init(testfiles.Locate("tableacl/test_table_tableacl_config.json"))
}
func createSetup(t *testing.T) (topo.Server, topo.Server) {
fromConn := fakezk.NewConn()
fromTS := zktopo.NewServer(fromConn)
toConn := fakezk.NewConn()
toTS := zktopo.NewServer(toConn)
for _, zkPath := range []string{"/zk/test_cell/vt", "/zk/global/vt"} {
if _, err := zk.CreateRecursive(fromConn, zkPath, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil {
t.Fatalf("cannot init fromTS: %v", err)
}
}
// create a keyspace and a couple tablets
if err := fromTS.CreateKeyspace("test_keyspace", &topo.Keyspace{}); err != nil {
t.Fatalf("cannot create keyspace: %v", err)
}
if err := fromTS.CreateShard("test_keyspace", "0", &topo.Shard{Cells: []string{"test_cell"}}); err != nil {
t.Fatalf("cannot create shard: %v", err)
}
if err := topo.CreateTablet(fromTS, &topo.Tablet{
Alias: topo.TabletAlias{
Cell: "test_cell",
Uid: 123,
},
Hostname: "masterhost",
Parent: topo.TabletAlias{},
IPAddr: "1.2.3.4",
Portmap: map[string]int{
"vt": 8101,
"vts": 8102,
"mysql": 3306,
},
Keyspace: "test_keyspace",
Shard: "0",
Type: topo.TYPE_MASTER,
State: topo.STATE_READ_WRITE,
DbNameOverride: "",
KeyRange: key.KeyRange{},
}); err != nil {
t.Fatalf("cannot create master tablet: %v", err)
}
if err := topo.CreateTablet(fromTS, &topo.Tablet{
Alias: topo.TabletAlias{
Cell: "test_cell",
Uid: 234,
},
IPAddr: "2.3.4.5",
Portmap: map[string]int{
"vt": 8101,
"vts": 8102,
"mysql": 3306,
},
Hostname: "slavehost",
Parent: topo.TabletAlias{
Cell: "test_cell",
Uid: 123,
},
Keyspace: "test_keyspace",
Shard: "0",
Type: topo.TYPE_REPLICA,
State: topo.STATE_READ_ONLY,
DbNameOverride: "",
KeyRange: key.KeyRange{},
}); err != nil {
t.Fatalf("cannot create slave tablet: %v", err)
}
os.Setenv("ZK_CLIENT_CONFIG", testfiles.Locate("topotools_test_zk_client.json"))
cells, err := fromTS.GetKnownCells()
if err != nil {
t.Fatalf("fromTS.GetKnownCells: %v", err)
}
log.Infof("Cells: %v", cells)
return fromTS, toTS
}
func createVindex() (Vindex, error) {
m := make(map[string]string)
m["json_path"] = testfiles.Locate("vtgate/numeric_static_map_test.json")
return CreateVindex("numeric_static_map", "numericStaticMap", m)
}