コード例 #1
0
ファイル: player.go プロジェクト: richarwu/vitess
func (client *client) StreamKeyRange(ctx context.Context, position string, keyspaceIdType key.KeyspaceIdType, keyRange *pb.KeyRange, charset *mproto.Charset) (chan *proto.BinlogTransaction, binlogplayer.ErrFunc, error) {
	req := &proto.KeyRangeRequest{
		Position:       position,
		KeyspaceIdType: keyspaceIdType,
		KeyRange:       key.ProtoToKeyRange(keyRange),
		Charset:        charset,
	}
	result := make(chan *proto.BinlogTransaction, 10)
	responseChan := make(chan *proto.BinlogTransaction, 10)
	resp := client.Client.StreamGo("UpdateStream.StreamKeyRange", req, responseChan)
	var finalError error
	go func() {
		defer close(result)
		for {
			select {
			case <-ctx.Done():
				finalError = ctx.Err()
				return
			case r, ok := <-responseChan:
				if !ok {
					// no more results from the server
					finalError = resp.Error
					return
				}
				result <- r
			}
		}
	}()
	return result, func() error {
		return finalError
	}, nil
}
コード例 #2
0
ファイル: proto3.go プロジェクト: ruiaylin/vitess
// ProtoToSrvKeyspace turns a proto to a Tablet
func ProtoToSrvKeyspace(s *pb.SrvKeyspace) *SrvKeyspace {
	result := &SrvKeyspace{
		Partitions:         make(map[TabletType]*KeyspacePartition),
		ShardingColumnName: s.ShardingColumnName,
		ShardingColumnType: key.ProtoToKeyspaceIdType(s.ShardingColumnType),
		SplitShardCount:    s.SplitShardCount,
	}
	for _, p := range s.Partitions {
		tt := ProtoToTabletType(p.ServedType)
		partition := &KeyspacePartition{}
		for _, sr := range p.ShardReferences {
			partition.ShardReferences = append(partition.ShardReferences, ShardReference{
				Name:     sr.Name,
				KeyRange: key.ProtoToKeyRange(sr.KeyRange),
			})
		}
		result.Partitions[tt] = partition
	}
	if len(s.ServedFrom) > 0 {
		result.ServedFrom = make(map[TabletType]string)
		for _, sf := range s.ServedFrom {
			tt := ProtoToTabletType(sf.TabletType)
			result.ServedFrom[tt] = sf.Keyspace
		}
	}
	return result
}
コード例 #3
0
ファイル: testing.go プロジェクト: richarwu/vitess
func newKeyRange(value string) key.KeyRange {
	_, result, err := topo.ValidateShardName(value)
	if err != nil {
		panic(err)
	}
	return key.ProtoToKeyRange(result)
}
コード例 #4
0
ファイル: tablet.go プロジェクト: haoqoo/vitess
// Complete validates and normalizes the tablet. If the shard name
// contains a '-' it is going to try to infer the keyrange from it.
func (tablet *Tablet) Complete() error {
	shard, kr, err := ValidateShardName(tablet.Shard)
	if err != nil {
		return err
	}
	tablet.Shard = shard
	tablet.KeyRange = key.ProtoToKeyRange(kr)
	return nil
}
コード例 #5
0
ファイル: row_splitter.go プロジェクト: richarwu/vitess
// NewRowSplitter returns a new row splitter for the given shard distribution.
func NewRowSplitter(shardInfos []*topo.ShardInfo, typ key.KeyspaceIdType, valueIndex int) *RowSplitter {
	result := &RowSplitter{
		Type:       typ,
		ValueIndex: valueIndex,
		KeyRanges:  make([]key.KeyRange, len(shardInfos)),
	}
	for i, si := range shardInfos {
		result.KeyRanges[i] = key.ProtoToKeyRange(si.KeyRange)
	}
	return result
}
コード例 #6
0
ファイル: vtgate_test.go プロジェクト: xgwubin/vitess
func TestVTGateSplitQuery(t *testing.T) {
	keyspace := "TestVTGateSplitQuery"
	keyranges, _ := key.ParseShardingSpec(DefaultShardSpec)
	s := createSandbox(keyspace)
	for _, kr := range keyranges {
		s.MapTestConn(key.KeyRangeString(kr), &sandboxConn{})
	}
	sql := "select col1, col2 from table"
	splitCount := 24
	result := new(proto.SplitQueryResult)
	err := rpcVTGate.SplitQuery(context.Background(),
		keyspace,
		sql,
		nil,
		"",
		splitCount,
		result)
	if err != nil {
		t.Errorf("want nil, got %v", err)
	}
	_, err = getAllShards(DefaultShardSpec)
	// Total number of splits should be number of shards * splitsPerShard
	if splitCount != len(result.Splits) {
		t.Errorf("wrong number of splits, want \n%+v, got \n%+v", splitCount, len(result.Splits))
	}
	actualSqlsByKeyRange := map[kproto.KeyRange][]string{}
	for _, split := range result.Splits {
		if split.Size != sandboxSQRowCount {
			t.Errorf("wrong split size, want \n%+v, got \n%+v", sandboxSQRowCount, split.Size)
		}
		if split.Query.Keyspace != keyspace {
			t.Errorf("wrong split size, want \n%+v, got \n%+v", keyspace, split.Query.Keyspace)
		}
		if len(split.Query.KeyRanges) != 1 {
			t.Errorf("wrong number of keyranges, want \n%+v, got \n%+v", 1, len(split.Query.KeyRanges))
		}
		if split.Query.TabletType != topo.TYPE_RDONLY {
			t.Errorf("wrong tablet type, want \n%+v, got \n%+v", topo.TYPE_RDONLY, split.Query.TabletType)
		}
		kr := split.Query.KeyRanges[0]
		actualSqlsByKeyRange[kr] = append(actualSqlsByKeyRange[kr], split.Query.Sql)
	}
	expectedSqlsByKeyRange := map[kproto.KeyRange][]string{}
	for _, kr := range keyranges {
		expectedSqlsByKeyRange[kproto.ProtoToKeyRange(kr)] = []string{
			"select col1, col2 from table /*split 0 */",
			"select col1, col2 from table /*split 1 */",
			"select col1, col2 from table /*split 2 */",
		}
	}
	if !reflect.DeepEqual(actualSqlsByKeyRange, expectedSqlsByKeyRange) {
		t.Errorf("splits contain the wrong sqls and/or keyranges, got: %v, want: %v", actualSqlsByKeyRange, expectedSqlsByKeyRange)
	}
}
コード例 #7
0
ファイル: streamer.go プロジェクト: zhzhy917/vitess
// StreamKeyRange is part of the pbs.UpdateStreamServer interface
func (server *UpdateStream) StreamKeyRange(req *pb.StreamKeyRangeRequest, stream pbs.UpdateStream_StreamKeyRangeServer) (err error) {
	defer server.updateStream.HandlePanic(&err)
	return server.updateStream.StreamKeyRange(&proto.KeyRangeRequest{
		Position:       myproto.ProtoToReplicationPosition(req.Position),
		KeyspaceIdType: key.ProtoToKeyspaceIdType(req.KeyspaceIdType),
		KeyRange:       key.ProtoToKeyRange(req.KeyRange),
		Charset:        mproto.ProtoToCharset(req.Charset),
	}, func(reply *proto.BinlogTransaction) error {
		return stream.Send(&pb.StreamKeyRangeResponse{
			BinlogTransaction: proto.BinlogTransactionToProto(reply),
		})
	})
}
コード例 #8
0
ファイル: topo_utils.go プロジェクト: cgvarela/vitess
// This maps a list of keyranges to shard names.
func resolveKeyRangeToShards(allShards []topo.ShardReference, kr *pb.KeyRange) ([]string, error) {
	shards := make([]string, 0, 1)

	if !key.KeyRangeIsPartial(kr) {
		for j := 0; j < len(allShards); j++ {
			shards = append(shards, allShards[j].Name)
		}
		return shards, nil
	}
	for j := 0; j < len(allShards); j++ {
		shard := allShards[j]
		if key.KeyRangesIntersect(key.ProtoToKeyRange(kr), shard.KeyRange) {
			shards = append(shards, shard.Name)
		}
	}
	return shards, nil
}
コード例 #9
0
ファイル: scatter_conn.go プロジェクト: xgwubin/vitess
// SplitQueryKeyRange scatters a SplitQuery request to all shards. For a set of
// splits received from a shard, it construct a KeyRange queries by
// appending that shard's keyrange to the splits. Aggregates all splits across
// all shards in no specific order and returns.
func (stc *ScatterConn) SplitQueryKeyRange(ctx context.Context, sql string, bindVariables map[string]interface{}, splitColumn string, splitCount int, keyRangeByShard map[string]*pb.KeyRange, keyspace string) ([]proto.SplitQueryPart, error) {
	tabletType := pb.TabletType_RDONLY
	actionFunc := func(shard string, transactionID int64, results chan<- interface{}) error {
		// Get all splits from this shard
		queries, err := stc.gateway.SplitQuery(ctx, keyspace, shard, tabletType, sql, bindVariables, splitColumn, splitCount)
		if err != nil {
			return err
		}
		// Append the keyrange for this shard to all the splits received
		keyranges := []kproto.KeyRange{kproto.ProtoToKeyRange(keyRangeByShard[shard])}
		splits := []proto.SplitQueryPart{}
		for _, query := range queries {
			krq := &proto.KeyRangeQuery{
				Sql:           query.Query.Sql,
				BindVariables: query.Query.BindVariables,
				Keyspace:      keyspace,
				KeyRanges:     keyranges,
				TabletType:    topo.TYPE_RDONLY,
			}
			split := proto.SplitQueryPart{
				Query: krq,
				Size:  query.RowCount,
			}
			splits = append(splits, split)
		}
		// Push all the splits from this shard to results channel
		results <- splits
		return nil
	}

	shards := []string{}
	for shard := range keyRangeByShard {
		shards = append(shards, shard)
	}
	allSplits, allErrors := stc.multiGo(ctx, "SplitQuery", keyspace, shards, tabletType, NewSafeSession(&proto.Session{}), false, actionFunc)
	splits := []proto.SplitQueryPart{}
	for s := range allSplits {
		splits = append(splits, s.([]proto.SplitQueryPart)...)
	}
	if allErrors.HasErrors() {
		err := allErrors.AggrError(stc.aggregateErrors)
		return nil, err
	}
	return splits, nil
}
コード例 #10
0
ファイル: proto3.go プロジェクト: afrolovskiy/vitess
// ProtoToTablet turns a proto to a Tablet
func ProtoToTablet(t *pb.Tablet) *Tablet {
	result := &Tablet{
		Alias:          ProtoToTabletAlias(t.Alias),
		Hostname:       t.Hostname,
		IPAddr:         t.Ip,
		Portmap:        make(map[string]int),
		Keyspace:       t.Keyspace,
		Shard:          t.Shard,
		KeyRange:       key.ProtoToKeyRange(t.KeyRange),
		Type:           ProtoToTabletType(t.Type),
		DbNameOverride: t.DbNameOverride,
		Tags:           t.Tags,
		Health:         t.HealthMap,
	}
	for k, v := range t.PortMap {
		result.Portmap[k] = int(v)
	}
	return result
}
コード例 #11
0
ファイル: tablet.go プロジェクト: cinderalla/vitess
// InitTablet creates or updates a tablet. If no parent is specified
// in the tablet, and the tablet has a slave type, we will find the
// appropriate parent. If createShardAndKeyspace is true and the
// parent keyspace or shard don't exist, they will be created.  If
// update is true, and a tablet with the same ID exists, update it.
// If Force is true, and a tablet with the same ID already exists, it
// will be scrapped and deleted, and then recreated.
func (wr *Wrangler) InitTablet(ctx context.Context, tablet *topo.Tablet, force, createShardAndKeyspace, update bool) error {
	if err := topo.TabletComplete(tablet); err != nil {
		return err
	}

	if topo.IsInReplicationGraph(tablet.Type) {
		// get the shard, possibly creating it
		var err error
		var si *topo.ShardInfo

		if createShardAndKeyspace {
			// create the parent keyspace and shard if needed
			si, err = topotools.GetOrCreateShard(ctx, wr.ts, tablet.Keyspace, tablet.Shard)
		} else {
			si, err = wr.ts.GetShard(ctx, tablet.Keyspace, tablet.Shard)
			if err == topo.ErrNoNode {
				return fmt.Errorf("missing parent shard, use -parent option to create it, or CreateKeyspace / CreateShard")
			}
		}

		// get the shard, checks a couple things
		if err != nil {
			return fmt.Errorf("cannot get (or create) shard %v/%v: %v", tablet.Keyspace, tablet.Shard, err)
		}
		if key.ProtoToKeyRange(si.KeyRange) != tablet.KeyRange {
			return fmt.Errorf("shard %v/%v has a different KeyRange: %v != %v", tablet.Keyspace, tablet.Shard, si.KeyRange, tablet.KeyRange)
		}
		if tablet.Type == topo.TYPE_MASTER && !topo.TabletAliasIsZero(si.MasterAlias) && topo.ProtoToTabletAlias(si.MasterAlias) != tablet.Alias && !force {
			return fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", si.MasterAlias, tablet.Keyspace, tablet.Shard)
		}

		// update the shard record if needed
		if err := wr.updateShardCellsAndMaster(ctx, si, topo.TabletAliasToProto(tablet.Alias), topo.TabletTypeToProto(tablet.Type), force); err != nil {
			return err
		}
	}

	err := topo.CreateTablet(ctx, wr.ts, tablet)
	if err != nil && err == topo.ErrNodeExists {
		// Try to update nicely, but if it fails fall back to force behavior.
		if update || force {
			oldTablet, err := wr.ts.GetTablet(ctx, tablet.Alias)
			if err != nil {
				wr.Logger().Warningf("failed reading tablet %v: %v", tablet.Alias, err)
			} else {
				if oldTablet.Keyspace == tablet.Keyspace && oldTablet.Shard == tablet.Shard {
					*(oldTablet.Tablet) = *tablet
					if err := topo.UpdateTablet(ctx, wr.ts, oldTablet); err != nil {
						wr.Logger().Warningf("failed updating tablet %v: %v", tablet.Alias, err)
						// now fall through the Scrap case
					} else {
						if !topo.IsInReplicationGraph(tablet.Type) {
							return nil
						}

						if err := topo.UpdateTabletReplicationData(ctx, wr.ts, tablet); err != nil {
							wr.Logger().Warningf("failed updating tablet replication data for %v: %v", tablet.Alias, err)
							// now fall through the Scrap case
						} else {
							return nil
						}
					}
				}
			}
		}
		if force {
			if err = wr.Scrap(ctx, tablet.Alias, force, false); err != nil {
				wr.Logger().Errorf("failed scrapping tablet %v: %v", tablet.Alias, err)
				return err
			}
			if err := wr.ts.DeleteTablet(ctx, tablet.Alias); err != nil {
				// we ignore this
				wr.Logger().Errorf("failed deleting tablet %v: %v", tablet.Alias, err)
			}
			return topo.CreateTablet(ctx, wr.ts, tablet)
		}
	}
	return err
}
コード例 #12
0
ファイル: rebuild.go プロジェクト: ruiaylin/vitess
// This function should only be used with an action lock on the keyspace
// - otherwise the consistency of the serving graph data can't be
// guaranteed.
//
// Take data from the global keyspace and rebuild the local serving
// copies in each cell.
func (wr *Wrangler) rebuildKeyspace(ctx context.Context, keyspace string, cells []string, rebuildSrvShards bool) error {
	wr.logger.Infof("rebuildKeyspace %v", keyspace)

	ki, err := wr.ts.GetKeyspace(ctx, keyspace)
	if err != nil {
		return err
	}

	var shardCache map[string]*topo.ShardInfo
	if rebuildSrvShards {
		shards, err := wr.ts.GetShardNames(ctx, keyspace)
		if err != nil {
			return nil
		}

		// Rebuild all shards in parallel, save the shards
		shardCache = make(map[string]*topo.ShardInfo)
		wg := sync.WaitGroup{}
		mu := sync.Mutex{}
		rec := concurrency.FirstErrorRecorder{}
		for _, shard := range shards {
			wg.Add(1)
			go func(shard string) {
				if shardInfo, err := wr.RebuildShardGraph(ctx, keyspace, shard, cells); err != nil {
					rec.RecordError(fmt.Errorf("RebuildShardGraph failed: %v/%v %v", keyspace, shard, err))
				} else {
					mu.Lock()
					shardCache[shard] = shardInfo
					mu.Unlock()
				}
				wg.Done()
			}(shard)
		}
		wg.Wait()
		if rec.HasErrors() {
			return rec.Error()
		}

	} else {
		shardCache, err = wr.ts.FindAllShardsInKeyspace(ctx, keyspace)
		if err != nil {
			return err
		}
	}

	// Build the list of cells to work on: we get the union
	// of all the Cells of all the Shards, limited to the provided cells.
	//
	// srvKeyspaceMap is a map:
	//   key: cell
	//   value: topo.SrvKeyspace object being built
	srvKeyspaceMap := make(map[string]*topo.SrvKeyspace)
	wr.findCellsForRebuild(ki, shardCache, cells, srvKeyspaceMap)

	// Then we add the cells from the keyspaces we might be 'ServedFrom'.
	for _, ksf := range ki.ServedFroms {
		servedFromShards, err := wr.ts.FindAllShardsInKeyspace(ctx, ksf.Keyspace)
		if err != nil {
			return err
		}
		wr.findCellsForRebuild(ki, servedFromShards, cells, srvKeyspaceMap)
	}

	// for each entry in the srvKeyspaceMap map, we do the following:
	// - read the SrvShard structures for each shard / cell
	// - if not present, build an empty one from global Shard
	// - compute the union of the db types (replica, master, ...)
	// - sort the shards in the list by range
	// - check the ranges are compatible (no hole, covers everything)
	for cell, srvKeyspace := range srvKeyspaceMap {
		srvKeyspace.Partitions = make(map[topo.TabletType]*topo.KeyspacePartition)
		for _, si := range shardCache {
			servedTypes := si.GetServedTypesPerCell(cell)

			// for each type this shard is supposed to serve,
			// add it to srvKeyspace.Partitions
			for _, tabletType := range servedTypes {
				if _, ok := srvKeyspace.Partitions[tabletType]; !ok {
					srvKeyspace.Partitions[tabletType] = &topo.KeyspacePartition{
						ShardReferences: make([]topo.ShardReference, 0),
					}
				}
				srvKeyspace.Partitions[tabletType].ShardReferences = append(srvKeyspace.Partitions[tabletType].ShardReferences, topo.ShardReference{
					Name:     si.ShardName(),
					KeyRange: key.ProtoToKeyRange(si.KeyRange),
				})
			}
		}

		if err := wr.orderAndCheckPartitions(cell, srvKeyspace); err != nil {
			return err
		}
	}

	// and then finally save the keyspace objects
	for cell, srvKeyspace := range srvKeyspaceMap {
		wr.logger.Infof("updating keyspace serving graph in cell %v for %v", cell, keyspace)
		if err := wr.ts.UpdateSrvKeyspace(ctx, cell, keyspace, srvKeyspace); err != nil {
			return fmt.Errorf("writing serving data failed: %v", err)
		}
	}
	return nil
}