func TestStream(t *testing.T) {
	env := setup("cat $3", 0)
	defer cleanup(env)

	var transactions []transaction

	out, err := ioutil.ReadFile(testfiles.Locate("mysqlctl_test/expected.json"))
	if err != nil {
		t.Fatal(err)
	}
	err = json.Unmarshal(out, &transactions)
	if err != nil {
		t.Fatal(err)
	}
	svm := &sync2.ServiceManager{}

	curTransaction := 0
	sendTx := func(tx *proto.BinlogTransaction) error {
		for i, stmt := range tx.Statements {
			if transactions[curTransaction].Statements[i].Sql != string(stmt.Sql) {
				t.Errorf("want %s, got %s", transactions[curTransaction].Statements[i].Sql, stmt.Sql)
			}
			if transactions[curTransaction].Statements[i].Category != stmt.Category {
				t.Errorf("want %d, got %d", transactions[curTransaction].Statements[i].Category, stmt.Category)
			}
		}
		if transactions[curTransaction].GTIDField != tx.GTIDField {
			t.Errorf("want %#v, got %#v", transactions[curTransaction].GTIDField, tx.GTIDField)
		}
		curTransaction++
		if curTransaction == len(transactions) {
			// Launch as goroutine to prevent deadlock.
			go svm.Stop()
		}
		// Uncomment the following lines to produce a different set of
		// expected outputs. You'll need to massage the file a bit afterwards.
		/*
			fmt.Printf("{\n\"Statements\": [\n")
			for i := 0; i < len(tx.Statements); i++ {
				fmt.Printf(`{"Category": %d, "Sql": %#v}`, tx.Statements[i].Category, string(tx.Statements[i].Sql))
				if i == len(tx.Statements)-1 {
					fmt.Printf("\n")
				} else {
					fmt.Printf(",\n")
				}
			}
			fmt.Printf("],\n")
			fmt.Printf("\"GTID\": \"%s\"\n},\n", tx.GTID)
		*/
		return nil
	}
	bls := newTestBinlogFileStreamer("db", testfiles.Locate("mysqlctl_test/vt-0000041983-bin"), myproto.ReplicationPosition{}, sendTx)
	svm.Go(func(ctx *sync2.ServiceContext) error {
		return bls.streamFilePos(ctx, "vt-0000041983-bin.000001", 0)
	})
	err = svm.Join()
	if err != nil {
		t.Error(err)
	}
}
Example #2
0
func TestStream(t *testing.T) {
	env := setup("cat $3", 0)
	defer cleanup(env)

	var transactions []transaction

	out, err := ioutil.ReadFile(testfiles.Locate("mysqlctl_test/expected.json"))
	if err != nil {
		t.Fatal(err)
	}
	err = json.Unmarshal(out, &transactions)
	if err != nil {
		t.Fatal(err)
	}

	curTransaction := 0
	bls := NewBinlogStreamer("db", testfiles.Locate("mysqlctl_test/vt-0000041983-bin"))
	err = bls.Stream("vt-0000041983-bin.000001", 0, func(tx *proto.BinlogTransaction) error {
		for i, stmt := range tx.Statements {
			if transactions[curTransaction].Statements[i].Sql != string(stmt.Sql) {
				t.Errorf("want %s, got %s", transactions[curTransaction].Statements[i].Sql, stmt.Sql)
			}
			if transactions[curTransaction].Statements[i].Category != stmt.Category {
				t.Errorf("want %d, got %d", transactions[curTransaction].Statements[i].Category, stmt.Category)
			}
		}
		if transactions[curTransaction].GroupId != tx.GroupId {
			t.Errorf("want %#v, got %#v", transactions[curTransaction].GroupId, tx.GroupId)
		}
		curTransaction++
		if curTransaction == len(transactions) {
			// Launch as goroutine to prevent deadlock.
			go bls.Stop()
		}
		// Uncomment the following lines to produce a different set of
		// expected outputs. You'll need to massage the file a bit afterwards.
		/*
			fmt.Printf("{\n\"Statements\": [\n")
			for i := 0; i < len(tx.Statements); i++ {
				fmt.Printf(`{"Category": %d, "Sql": %#v}`, tx.Statements[i].Category, string(tx.Statements[i].Sql))
				if i == len(tx.Statements)-1 {
					fmt.Printf("\n")
				} else {
					fmt.Printf(",\n")
				}
			}
			fmt.Printf("],\n")
			fmt.Printf("\"GroupId\": \"%s\"\n},\n", tx.GroupId)
		*/
		return nil
	})
	if err != nil {
		t.Error(err)
	}
}
Example #3
0
func TestFromFile(t *testing.T) {
	conn := NewConnFromFile(testfiles.Locate("fakezk_test_config.json"))

	keyspaces, _, err := conn.Children("/zk/testing/vt/ns")
	if err != nil {
		t.Errorf("conn.Children: %v", err)
	}
	if len(keyspaces) != 1 || keyspaces[0] != "test_keyspace" {
		t.Errorf("conn.Children returned bad value: %v", keyspaces)
	}

	data, _, err := conn.Get("/zk/testing/vt/ns/test_keyspace")
	if err != nil {
		t.Errorf("conn.Get(/zk/testing/vt/ns/test_keyspace): %v", err)
	}
	if !strings.Contains(string(data), "ShardReferences") {
		t.Errorf("conn.Get(/zk/testing/vt/ns/test_keyspace) returned bad value: %v", data)
	}

	data, _, err = conn.Get("/zk/testing/vt/ns/test_keyspace/0/master")
	if err != nil {
		t.Errorf("conn.Get(/zk/testing/vt/ns/test_keyspace/0/master): %v", err)
	}
	if !strings.Contains(string(data), "NamedPortMap") {
		t.Errorf("conn.Get(/zk/testing/vt/ns/test_keyspace/0/master) returned bad value: %v", data)
	}
}
Example #4
0
func iterateFile(name string) (testCaseIterator chan testCase) {
	name = testfiles.Locate("sqlparser_test/" + name)
	fd, err := os.OpenFile(name, os.O_RDONLY, 0)
	if err != nil {
		panic(fmt.Sprintf("Could not open file %s", name))
	}
	testCaseIterator = make(chan testCase)
	go func() {
		defer close(testCaseIterator)

		r := bufio.NewReader(fd)
		lineno := 0
		for {
			line, err := r.ReadString('\n')
			lines := strings.Split(strings.TrimRight(line, "\n"), "#")
			lineno++
			if err != nil {
				if err != io.EOF {
					panic(fmt.Sprintf("Error reading file %s: %s", name, err.Error()))
				}
				break
			}
			input := lines[0]
			output := ""
			if len(lines) > 1 {
				output = lines[1]
			}
			if input == "" {
				continue
			}
			testCaseIterator <- testCase{lineno, input, output}
		}
	}()
	return testCaseIterator
}
Example #5
0
func newSpecialReader(t *testing.T, filename string) *specialReader {
	filename = testfiles.Locate(filename)
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		t.Fatalf("Cannot read file %v: %v", filename, err)
	}
	return &specialReader{t, b, 0}
}
Example #6
0
func TestInitWithInvalidConfigFile(t *testing.T) {
	setUpTableACL(&simpleacl.Factory{})
	defer func() {
		err := recover()
		if err == nil {
			t.Fatalf("init should fail for an invalid config file")
		}
	}()
	Init(testfiles.Locate("tableacl/invalid_tableacl_config.json"))
}
Example #7
0
func iterateJSONFile(name string) (testCaseIterator chan testCase) {
	name = testfiles.Locate("sqlparser_test/" + name)
	fd, err := os.OpenFile(name, os.O_RDONLY, 0)
	if err != nil {
		panic(fmt.Sprintf("Could not open file %s", name))
	}
	testCaseIterator = make(chan testCase)
	go func() {
		defer close(testCaseIterator)

		r := bufio.NewReader(fd)
		lineno := 0
		for {
			binput, _, err := r.ReadLine()
			input := string(binput)
			lineno++
			if err != nil {
				if err != io.EOF {
					panic(fmt.Sprintf("Error reading file %s: %s", name, err.Error()))
				}
				break
			}
			if input == "" || input[0] == '#' {
				//fmt.Printf("%s\n", input)
				continue
			}
			var output []byte
			for {
				l, err := r.ReadBytes('\n')
				lineno++
				if err != nil {
					panic(fmt.Sprintf("Error reading file %s: %s", name, err.Error()))
				}
				output = append(output, l...)
				if l[0] == '}' {
					output = output[:len(output)-1]
					b := bytes.NewBuffer(make([]byte, 0, 64))
					if err := json.Compact(b, output); err == nil {
						output = b.Bytes()
					}
					break
				}
				if l[0] == '"' {
					output = output[1 : len(output)-2]
					break
				}
			}
			testCaseIterator <- testCase{lineno, input, string(output)}
		}
	}()
	return testCaseIterator
}
// TestRoration should not hang
func TestRotation(t *testing.T) {
	env := setup("cat $3", 0)
	defer cleanup(env)

	bls := NewBinlogStreamer("db", testfiles.Locate("mysqlctl_test/vt-0000041983-bin"))
	err := bls.Stream("vt-0000041983-bin.000004", 2682, func(tx *proto.BinlogTransaction) error {
		bls.Stop()
		return nil
	})
	if err != nil {
		t.Error(err)
	}
}
Example #9
0
func TestSimple(t *testing.T) {
	input := testfiles.Locate("bson_test/simple_type.go")
	b, err := ioutil.ReadFile(input)
	if err != nil {
		log.Fatal(err)
	}
	out, err := generateCode(string(b), "MyType")
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		return
	}
	fmt.Printf("%s\n", out)
}
// TestRoration should not hang
func TestRotation(t *testing.T) {
	env := setup("cat $3", 0)
	defer cleanup(env)

	bls := newTestBinlogFileStreamer("db", testfiles.Locate("mysqlctl_test/vt-0000041983-bin"))
	err := bls.streamFilePos("vt-0000041983-bin.000004", 2682, func(tx *proto.BinlogTransaction) error {
		// Launch as goroutine to prevent deadlock.
		go bls.Stop()
		return nil
	})
	if err != nil {
		t.Error(err)
	}
}
// TestRoration should not hang
func TestRotation(t *testing.T) {
	env := setup("cat $3", 0)
	defer cleanup(env)

	svm := &sync2.ServiceManager{}
	bls := newTestBinlogFileStreamer("db", testfiles.Locate("mysqlctl_test/vt-0000041983-bin"), myproto.ReplicationPosition{}, func(tx *proto.BinlogTransaction) error {
		// Launch as goroutine to prevent deadlock.
		go svm.Stop()
		return nil
	})
	svm.Go(func(ctx *sync2.ServiceContext) error {
		return bls.streamFilePos(ctx, "vt-0000041983-bin.000004", 2682)
	})
	err := svm.Join()
	if err != nil {
		t.Error(err)
	}
}
Example #12
0
func main() {
	input := testfiles.Locate("bson_test/simple_type.go")
	b, err := ioutil.ReadFile(input)
	if err != nil {
		log.Fatal(err)
	}
	src := string(b)

	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, "", src, 0)
	if err != nil {
		log.Fatal(err)
	}
	//ast.Print(fset, f)
	typeInfo, err := FindType(f, "MyType")
	if err != nil {
		fmt.Println(err)
		return
	}
	generator.ExecuteTemplate(os.Stdout, "Body", typeInfo)
}
Example #13
0
func BenchmarkFileStreamerParseEvents(b *testing.B) {
	filename := testfiles.Locate("binlog_test/vt-0000062347-bin.000001")
	var svm sync2.ServiceManager
	count := 0
	bls := newTestBinlogFileStreamer("vt_test_database", "", myproto.ReplicationPosition{}, func(tx *proto.BinlogTransaction) error {
		count++
		return nil
	})

	for i := 0; i < b.N; i++ {
		if err := bls.file.Init(filename, 0); err != nil {
			b.Fatalf("%v", err)
		}
		svm.Go(bls.run)
		if err := svm.Join(); err != nil {
			b.Errorf("%v", err)
		}
		bls.file.Close()
	}

	b.Logf("%d transactions processed", count)
}
Example #14
0
func BenchmarkConnStreamerParseEvents(b *testing.B) {
	filename := testfiles.Locate("binlog_test/vt-0000062347-bin.000001")
	var svm sync2.ServiceManager
	count := 0
	bls := &binlogConnStreamer{dbname: "vt_test_database", sendTransaction: func(tx *proto.BinlogTransaction) error {
		count++
		return nil
	}}

	for i := 0; i < b.N; i++ {
		events := readEvents(b, filename)
		svm.Go(func(svc *sync2.ServiceContext) error {
			_, err := bls.parseEvents(svc, events)
			return err
		})
		if err := svm.Join(); err != ServerEOF {
			b.Errorf("%v", err)
		}
	}

	b.Logf("%d transactions processed", count)
}
Example #15
0
func readLines(t *testing.T, name string) []string {
	file, err := os.Open(testfiles.Locate(name))
	if err != nil {
		t.Fatalf("Cannot open %v: %v", name, err)
	}
	r := NewCSVReader(file, ',')

	lines := make([]string, 0)

	for {
		line, err := r.ReadRecord()
		if err == io.EOF {
			break
		}
		if err != nil {
			t.Fatalf("Unexpected error: %v", err)
		}
		lines = append(lines, string(line))
	}

	return lines
}
func readData(t *testing.T, name string, numberColumn bool) []pair {
	file, err := os.Open(testfiles.Locate(name))
	if err != nil {
		t.Fatalf("Cannot open %v: %v", name, err)
	}
	r := NewKeyspaceCSVReader(file, ',', numberColumn)

	keyspaceIds := make([]pair, 0)

	for {
		kid, line, err := r.ReadRecord()
		if err == io.EOF {
			break
		}
		if err != nil {
			t.Fatalf("Unexpected error: %v", err)
		}
		keyspaceIds = append(keyspaceIds, pair{kid, string(line)})
	}

	return keyspaceIds
}
Example #17
0
func locateFile(name string) string {
	if path.IsAbs(name) {
		return name
	}
	return testfiles.Locate("vtgate/" + name)
}
Example #18
0
func locateFile(name string) string {
	if path.IsAbs(name) {
		return name
	}
	return testfiles.Locate("sqlparser_test/" + name)
}
Example #19
0
func createSetup(ctx context.Context, t *testing.T) (topo.Impl, topo.Impl) {
	fromConn := fakezk.NewConn()
	fromTS := zktopo.NewServer(fromConn)

	toConn := fakezk.NewConn()
	toTS := zktopo.NewServer(toConn)

	for _, zkPath := range []string{"/zk/test_cell/vt", "/zk/global/vt"} {
		if _, err := zk.CreateRecursive(fromConn, zkPath, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil {
			t.Fatalf("cannot init fromTS: %v", err)
		}
	}

	// create a keyspace and a couple tablets
	if err := fromTS.CreateKeyspace(ctx, "test_keyspace", &topodatapb.Keyspace{}); err != nil {
		t.Fatalf("cannot create keyspace: %v", err)
	}
	if err := fromTS.CreateShard(ctx, "test_keyspace", "0", &topodatapb.Shard{Cells: []string{"test_cell"}}); err != nil {
		t.Fatalf("cannot create shard: %v", err)
	}
	tts := topo.Server{Impl: fromTS}
	if err := tts.CreateTablet(ctx, &topodatapb.Tablet{
		Alias: &topodatapb.TabletAlias{
			Cell: "test_cell",
			Uid:  123,
		},
		Hostname: "masterhost",
		Ip:       "1.2.3.4",
		PortMap: map[string]int32{
			"vt":    8101,
			"gprc":  8102,
			"mysql": 3306,
		},
		Keyspace:       "test_keyspace",
		Shard:          "0",
		Type:           topodatapb.TabletType_MASTER,
		DbNameOverride: "",
		KeyRange:       nil,
	}); err != nil {
		t.Fatalf("cannot create master tablet: %v", err)
	}
	if err := tts.CreateTablet(ctx, &topodatapb.Tablet{
		Alias: &topodatapb.TabletAlias{
			Cell: "test_cell",
			Uid:  234,
		},
		Ip: "2.3.4.5",
		PortMap: map[string]int32{
			"vt":    8101,
			"grpc":  8102,
			"mysql": 3306,
		},
		Hostname: "slavehost",

		Keyspace:       "test_keyspace",
		Shard:          "0",
		Type:           topodatapb.TabletType_REPLICA,
		DbNameOverride: "",
		KeyRange:       nil,
	}); err != nil {
		t.Fatalf("cannot create slave tablet: %v", err)
	}

	os.Setenv("ZK_CLIENT_CONFIG", testfiles.Locate("topo_helpers_test_zk_client.json"))
	cells, err := fromTS.GetKnownCells(ctx)
	if err != nil {
		t.Fatalf("fromTS.GetKnownCells: %v", err)
	}
	log.Infof("Cells: %v", cells)

	return fromTS, toTS
}
Example #20
0
func locateFile(name string) string {
	if path.IsAbs(name) {
		return name
	}
	return testfiles.Locate("tabletserver/" + name)
}
Example #21
0
func TestInitWithValidConfig(t *testing.T) {
	setUpTableACL(&simpleacl.Factory{})
	Init(testfiles.Locate("tableacl/test_table_tableacl_config.json"))
}
Example #22
0
func createSetup(t *testing.T) (topo.Server, topo.Server) {
	fromConn := fakezk.NewConn()
	fromTS := zktopo.NewServer(fromConn)

	toConn := fakezk.NewConn()
	toTS := zktopo.NewServer(toConn)

	for _, zkPath := range []string{"/zk/test_cell/vt", "/zk/global/vt"} {
		if _, err := zk.CreateRecursive(fromConn, zkPath, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil {
			t.Fatalf("cannot init fromTS: %v", err)
		}
	}

	// create a keyspace and a couple tablets
	if err := fromTS.CreateKeyspace("test_keyspace", &topo.Keyspace{}); err != nil {
		t.Fatalf("cannot create keyspace: %v", err)
	}
	if err := fromTS.CreateShard("test_keyspace", "0", &topo.Shard{Cells: []string{"test_cell"}}); err != nil {
		t.Fatalf("cannot create shard: %v", err)
	}
	if err := topo.CreateTablet(fromTS, &topo.Tablet{
		Alias: topo.TabletAlias{
			Cell: "test_cell",
			Uid:  123,
		},
		Hostname: "masterhost",
		Parent:   topo.TabletAlias{},
		IPAddr:   "1.2.3.4",
		Portmap: map[string]int{
			"vt":    8101,
			"vts":   8102,
			"mysql": 3306,
		},
		Keyspace:       "test_keyspace",
		Shard:          "0",
		Type:           topo.TYPE_MASTER,
		State:          topo.STATE_READ_WRITE,
		DbNameOverride: "",
		KeyRange:       key.KeyRange{},
	}); err != nil {
		t.Fatalf("cannot create master tablet: %v", err)
	}
	if err := topo.CreateTablet(fromTS, &topo.Tablet{
		Alias: topo.TabletAlias{
			Cell: "test_cell",
			Uid:  234,
		},
		IPAddr: "2.3.4.5",
		Portmap: map[string]int{
			"vt":    8101,
			"vts":   8102,
			"mysql": 3306,
		},
		Hostname: "slavehost",

		Parent: topo.TabletAlias{
			Cell: "test_cell",
			Uid:  123,
		},
		Keyspace:       "test_keyspace",
		Shard:          "0",
		Type:           topo.TYPE_REPLICA,
		State:          topo.STATE_READ_ONLY,
		DbNameOverride: "",
		KeyRange:       key.KeyRange{},
	}); err != nil {
		t.Fatalf("cannot create slave tablet: %v", err)
	}

	os.Setenv("ZK_CLIENT_CONFIG", testfiles.Locate("topotools_test_zk_client.json"))
	cells, err := fromTS.GetKnownCells()
	if err != nil {
		t.Fatalf("fromTS.GetKnownCells: %v", err)
	}
	log.Infof("Cells: %v", cells)

	return fromTS, toTS
}
Example #23
0
func createVindex() (Vindex, error) {
	m := make(map[string]string)
	m["json_path"] = testfiles.Locate("vtgate/numeric_static_map_test.json")
	return CreateVindex("numeric_static_map", "numericStaticMap", m)
}