Example #1
0
func main() {
	flag.Parse()
	servenv.Init()
	defer servenv.Close()

	keyRange, err := key.ParseKeyRangeParts(*start, *end)
	if err != nil {
		log.Fatalf("Invalid key range: %v", err)
	}

	if *dbConfigFile == "" {
		log.Fatalf("Cannot start without db-config-file")
	}
	dbConfig, err := readDbConfig(*dbConfigFile)
	if err != nil {
		log.Fatalf("Cannot read db config file: %v", err)
	}

	var t []string
	if *tables != "" {
		t = strings.Split(*tables, ",")
		for i, table := range t {
			t[i] = strings.TrimSpace(table)
		}
	}

	interrupted := make(chan struct{})
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGTERM)
	go func() {
		for _ = range c {
			close(interrupted)
		}
	}()

	var vtClient mysqlctl.VtClient
	vtClient = mysqlctl.NewDbClient(dbConfig)
	err = vtClient.Connect()
	if err != nil {
		log.Fatalf("error in initializing dbClient: %v", err)
	}
	brs, err := mysqlctl.ReadStartPosition(vtClient, uint32(*uid))
	if err != nil {
		log.Fatalf("Cannot read start position from db: %v", err)
	}
	if *debug {
		vtClient = mysqlctl.NewDummyVtClient()
	}
	blp, err := mysqlctl.NewBinlogPlayer(vtClient, keyRange, uint32(*uid), brs, t, *txnBatch, time.Duration(*maxTxnInterval)*time.Second, *execDdl)
	if err != nil {
		log.Fatalf("error in initializing binlog player: %v", err)
	}
	err = blp.ApplyBinlogEvents(interrupted)
	if err != nil {
		log.Errorf("Error in applying binlog events, err %v", err)
	}
	log.Infof("vt_binlog_player done")
}
Example #2
0
func (bpc *BinlogPlayerController) Iteration() (err error) {
	defer func() {
		if x := recover(); x != nil {
			log.Errorf("%v: caught panic: %v", bpc, x)
			err = fmt.Errorf("panic: %v", x)
		}
	}()

	// Enable any user to set the timestamp.
	// We do it on every iteration to be sure, in case mysql was
	// restarted.
	bpc.DisableSuperToSetTimestamp()

	// create the db connection, connect it
	vtClient := mysqlctl.NewDbClient(bpc.dbConfig)
	if err := vtClient.Connect(); err != nil {
		return fmt.Errorf("can't connect to database: %v", err)
	}
	defer vtClient.Close()

	// Read the start position
	startPosition, err := mysqlctl.ReadStartPosition(vtClient, bpc.sourceShard.Uid)
	if err != nil {
		return fmt.Errorf("can't read startPosition: %v", err)
	}

	// Find the server list for the source shard in our cell
	addrs, err := bpc.ts.GetEndPoints(bpc.cell, bpc.sourceShard.Keyspace, bpc.sourceShard.Shard, topo.TYPE_REPLICA)
	if err != nil {
		return fmt.Errorf("can't find any source tablet for %v %v %v: %v", bpc.cell, bpc.sourceShard.String(), topo.TYPE_REPLICA, err)
	}
	if len(addrs.Entries) == 0 {
		return fmt.Errorf("empty source tablet list for %v %v %v", bpc.cell, bpc.sourceShard.String(), topo.TYPE_REPLICA)
	}
	newServerIndex := rand.Intn(len(addrs.Entries))
	addr := fmt.Sprintf("%v:%v", addrs.Entries[newServerIndex].Host, addrs.Entries[newServerIndex].NamedPortMap["_vtocc"])

	// check which kind of replication we're doing, tables or keyrange
	if len(bpc.sourceShard.Tables) > 0 {
		// tables, just get them
		player := mysqlctl.NewBinlogPlayerTables(vtClient, addr, bpc.sourceShard.Tables, startPosition, bpc.stopAtGroupId)
		return player.ApplyBinlogEvents(bpc.interrupted)
	} else {
		// the data we have to replicate is the intersection of the
		// source keyrange and our keyrange
		overlap, err := key.KeyRangesOverlap(bpc.sourceShard.KeyRange, bpc.keyRange)
		if err != nil {
			return fmt.Errorf("Source shard %v doesn't overlap destination shard %v", bpc.sourceShard.KeyRange, bpc.keyRange)
		}

		player := mysqlctl.NewBinlogPlayerKeyRange(vtClient, addr, overlap, startPosition, bpc.stopAtGroupId)
		return player.ApplyBinlogEvents(bpc.interrupted)
	}
}
Example #3
0
func (bpc *BinlogPlayerController) Iteration() (err error) {
	defer func() {
		if x := recover(); x != nil {
			log.Errorf("BinlogPlayerController caught panic: %v", x)
			err = fmt.Errorf("panic: %v", x)
		}
	}()
	bpc.states.SetState(BINLOG_PLAYER_CONNECTING)

	// create the db connection, connect it
	vtClient := mysqlctl.NewDbClient(bpc.dbConfig)
	if err := vtClient.Connect(); err != nil {
		return fmt.Errorf("can't connect to database: %v", err)
	}
	defer vtClient.Close()

	// Read the start position
	startPosition, err := mysqlctl.ReadStartPosition(vtClient, string(bpc.source.KeyRange.Start.Hex()), string(bpc.source.KeyRange.End.Hex()))
	if err != nil {
		return fmt.Errorf("can't read startPosition: %v", err)
	}

	// TODO(alainjobart): Find the server list

	// TODO(alainjobart): Pick a server (same if it's available,
	// if not clear master file / pos and keep only group id)

	// Create the player.
	bpc.mu.Lock()
	bpc.player, err = mysqlctl.NewBinlogPlayer(vtClient, startPosition, nil /*tables*/, 1 /*txnBatch*/, 30*time.Second /*maxTxnInterval*/, false /*execDdl*/)
	bpc.mu.Unlock()
	if err != nil {
		return fmt.Errorf("can't create player: %v", err)
	}

	// Run player loop until it's done.
	bpc.states.SetState(BINLOG_PLAYER_PLAYING)
	err = bpc.player.ApplyBinlogEvents(bpc.interrupted)

	bpc.mu.Lock()
	bpc.player = nil
	bpc.mu.Unlock()

	return err
}
Example #4
0
// BlpPositionList returns the current position of all the players
func (blm *BinlogPlayerMap) BlpPositionList() (*myproto.BlpPositionList, error) {
	// create a db connection for this purpose
	vtClient := mysqlctl.NewDbClient(&blm.dbConfig)
	if err := vtClient.Connect(); err != nil {
		return nil, fmt.Errorf("can't connect to database: %v", err)
	}
	defer vtClient.Close()

	result := &myproto.BlpPositionList{}
	blm.mu.Lock()
	defer blm.mu.Unlock()
	for _, bpc := range blm.players {
		blp, err := bpc.BlpPosition(vtClient)
		if err != nil {
			return nil, fmt.Errorf("can't read current position for %v: %v", bpc, err)
		}

		result.Entries = append(result.Entries, *blp)
	}
	return result, nil
}
Example #5
0
func (bpc *BinlogPlayerController) Iteration() (err error) {
	defer func() {
		if x := recover(); x != nil {
			log.Errorf("%v: Caught panic: %v", bpc, x)
			err = fmt.Errorf("panic: %v", x)
		}
	}()

	// Enable any user to set the timestamp.
	// We do it on every iteration to be sure, in case mysql was
	// restarted.
	bpc.DisableSuperToSetTimestamp()

	// create the db connection, connect it
	vtClient := mysqlctl.NewDbClient(bpc.dbConfig)
	if err := vtClient.Connect(); err != nil {
		return fmt.Errorf("can't connect to database: %v", err)
	}
	defer vtClient.Close()

	// Read the start position
	startPosition, err := mysqlctl.ReadStartPosition(vtClient, bpc.sourceShard.Uid)
	if err != nil {
		return fmt.Errorf("can't read startPosition: %v", err)
	}

	// Find the server list for the source shard in our cell
	addrs, err := bpc.ts.GetEndPoints(bpc.cell, bpc.sourceShard.Keyspace, bpc.sourceShard.Shard, topo.TYPE_REPLICA)
	if err != nil {
		return fmt.Errorf("can't find any source tablet for %v %v %v: %v", bpc.cell, bpc.sourceShard.String(), topo.TYPE_REPLICA, err)
	}
	if len(addrs.Entries) == 0 {
		return fmt.Errorf("empty source tablet list for %v %v %v", bpc.cell, bpc.sourceShard.String(), topo.TYPE_REPLICA)
	}

	// if the server we were using before is in the list, just keep using it
	usePreviousServer := false
	for _, addr := range addrs.Entries {
		vtAddr := fmt.Sprintf("%v:%v", addr.Host, addr.NamedPortMap["_vtocc"])
		if vtAddr == startPosition.Addr {
			log.Infof("%v: Previous server %s still healthy, using it", bpc, vtAddr)
			usePreviousServer = true
		}
	}

	// if we can't use the previous server, pick a new one randomly
	if !usePreviousServer {
		newServerIndex := rand.Intn(len(addrs.Entries))
		startPosition.Addr = fmt.Sprintf("%v:%v", addrs.Entries[newServerIndex].Host, addrs.Entries[newServerIndex].NamedPortMap["_vtocc"])
		startPosition.Position.MasterFilename = ""
		startPosition.Position.MasterPosition = 0
		log.Infof("%v: Connecting to different server: %s", bpc, startPosition.Addr)
	}

	// the data we have to replicate is the intersection of the
	// source keyrange and our keyrange
	overlap, err := key.KeyRangesOverlap(bpc.sourceShard.KeyRange, bpc.keyRange)
	if err != nil {
		return fmt.Errorf("Source shard %v doesn't overlap destination shard %v", bpc.sourceShard.KeyRange, bpc.keyRange)
	}

	// Create the player.
	player, err := mysqlctl.NewBinlogPlayer(vtClient, overlap, bpc.sourceShard.Uid, startPosition, nil /*tables*/, 1 /*txnBatch*/, 30*time.Second /*maxTxnInterval*/, false /*execDdl*/)
	if err != nil {
		return fmt.Errorf("can't create player: %v", err)
	}

	// Run player loop until it's done.
	return player.ApplyBinlogEvents(bpc.interrupted)
}