func (l *logger) LogMethodInvocation(methodInvocation *MethodInvocation) {
if methodInvocation.Error != "" {
protolog.Error(methodInvocation)
} else {
protolog.Info(methodInvocation)
}
}
func (a *discoveryAddresser) AssignRoles(cancel chan bool) (retErr error) {
protolog.Info(&log.StartAssignRoles{})
defer func() {
protolog.Info(&log.FinishAssignRoles{errorToString(retErr)})
}()
var version int64
oldServers := make(map[string]bool)
oldRoles := make(map[string]*proto.ServerRole)
oldMasters := make(map[uint64]string)
oldReplicas := make(map[uint64][]string)
var oldMinVersion int64
// Reconstruct state from a previous run
serverRoles, err := a.discoveryClient.GetAll(a.serverRoleDir())
if err != nil {
return err
}
for _, encodedServerRole := range serverRoles {
serverRole, err := decodeServerRole(encodedServerRole)
if err != nil {
return err
}
if oldServerRole, ok := oldRoles[serverRole.Id]; !ok || oldServerRole.Version < serverRole.Version {
oldRoles[serverRole.Id] = serverRole
oldServers[serverRole.Id] = true
}
if version < serverRole.Version+1 {
version = serverRole.Version + 1
}
}
for _, oldServerRole := range oldRoles {
for shard := range oldServerRole.Masters {
oldMasters[shard] = oldServerRole.Id
}
for shard := range oldServerRole.Replicas {
oldReplicas[shard] = append(oldReplicas[shard], oldServerRole.Id)
}
}
err = a.discoveryClient.WatchAll(a.serverStateDir(), cancel,
func(encodedServerStates map[string]string) error {
if len(encodedServerStates) == 0 {
return nil
}
newServerStates := make(map[string]*proto.ServerState)
shardLocations := make(map[uint64][]string)
newRoles := make(map[string]*proto.ServerRole)
newMasters := make(map[uint64]string)
newReplicas := make(map[uint64][]string)
masterRolesPerServer := a.sharder.NumShards() / uint64(len(encodedServerStates))
masterRolesRemainder := a.sharder.NumShards() % uint64(len(encodedServerStates))
replicaRolesPerServer := (a.sharder.NumShards() * (a.sharder.NumReplicas())) / uint64(len(encodedServerStates))
replicaRolesRemainder := (a.sharder.NumShards() * (a.sharder.NumReplicas())) % uint64(len(encodedServerStates))
for _, encodedServerState := range encodedServerStates {
serverState, err := decodeServerState(encodedServerState)
if err != nil {
return err
}
newServerStates[serverState.Id] = serverState
newRoles[serverState.Id] = &proto.ServerRole{
Id: serverState.Id,
Version: version,
Masters: make(map[uint64]bool),
Replicas: make(map[uint64]bool),
}
for shard := range serverState.Shards {
shardLocations[shard] = append(shardLocations[shard], serverState.Id)
}
}
// See if there's any roles we can delete
minVersion := int64(math.MaxInt64)
for _, serverState := range newServerStates {
if serverState.Version < minVersion {
minVersion = serverState.Version
}
}
// Delete roles that no servers are using anymore
if minVersion > oldMinVersion {
oldMinVersion = minVersion
serverRoles, err := a.discoveryClient.GetAll(a.serverRoleDir())
if err != nil {
return err
}
for key, encodedServerRole := range serverRoles {
serverRole, err := decodeServerRole(encodedServerRole)
if err != nil {
return err
}
if serverRole.Version < minVersion {
if err := a.discoveryClient.Delete(key); err != nil {
return err
}
protolog.Info(&log.DeleteServerRole{serverRole})
}
}
}
// if the servers are identical to last time then we know we'll
// assign shards the same way
if sameServers(oldServers, newServerStates) {
return nil
}
Master:
for shard := uint64(0); shard < a.sharder.NumShards(); shard++ {
if id, ok := oldMasters[shard]; ok {
if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) {
continue Master
}
}
for _, id := range oldReplicas[shard] {
if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) {
continue Master
}
}
for _, id := range shardLocations[shard] {
if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) {
continue Master
}
}
for id := range newServerStates {
if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) {
continue Master
}
}
protolog.Error(&log.FailedToAssignRoles{
ServerStates: newServerStates,
NumShards: a.sharder.NumShards(),
NumReplicas: a.sharder.NumReplicas(),
})
return nil
}
for replica := uint64(0); replica < a.sharder.NumReplicas(); replica++ {
Replica:
for shard := uint64(0); shard < a.sharder.NumShards(); shard++ {
if id, ok := oldMasters[shard]; ok {
if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) {
continue Replica
}
}
for _, id := range oldReplicas[shard] {
if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) {
continue Replica
}
}
for _, id := range shardLocations[shard] {
if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) {
continue Replica
}
}
for id := range newServerStates {
if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) {
continue Replica
}
}
for id := range newServerStates {
if swapReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer) {
continue Replica
}
}
protolog.Error(&log.FailedToAssignRoles{
ServerStates: newServerStates,
NumShards: a.sharder.NumShards(),
NumReplicas: a.sharder.NumReplicas(),
})
return nil
}
}
addresses := proto.Addresses{
Version: version,
Addresses: make(map[uint64]*proto.ShardAddresses),
}
for shard := uint64(0); shard < a.sharder.NumShards(); shard++ {
addresses.Addresses[shard] = &proto.ShardAddresses{Replicas: make(map[string]bool)}
}
for id, serverRole := range newRoles {
encodedServerRole, err := marshaler.MarshalToString(serverRole)
if err != nil {
return err
}
if err := a.discoveryClient.Set(a.serverRoleKeyVersion(id, version), encodedServerRole, 0); err != nil {
return err
}
protolog.Info(&log.SetServerRole{serverRole})
address := newServerStates[id].Address
for shard := range serverRole.Masters {
shardAddresses := addresses.Addresses[shard]
shardAddresses.Master = address
addresses.Addresses[shard] = shardAddresses
}
for shard := range serverRole.Replicas {
shardAddresses := addresses.Addresses[shard]
shardAddresses.Replicas[address] = true
addresses.Addresses[shard] = shardAddresses
}
}
encodedAddresses, err := marshaler.MarshalToString(&addresses)
if err != nil {
return err
}
if err := a.discoveryClient.Set(a.addressesKey(version), encodedAddresses, 0); err != nil {
return err
}
protolog.Info(&log.SetAddresses{&addresses})
version++
oldServers = make(map[string]bool)
for id := range newServerStates {
oldServers[id] = true
}
oldRoles = newRoles
oldMasters = newMasters
oldReplicas = newReplicas
return nil
})
if err == discovery.ErrCancelled {
return ErrCancelled
}
return err
}
// Error logs an RPC call at the error level.
func Error(serviceName string, methodName string, request proto.Message, response proto.Message, err error, duration time.Duration) {
protolog.Error(event(serviceName, methodName, request, response, err, duration))
}
// Serve serves stuff.
func Serve(
port uint16,
registerFunc func(*grpc.Server),
opts ServeOptions,
) (retErr error) {
start := time.Now()
defer func() {
if retErr != nil {
protolog.Error(
&ServerFinished{
Error: retErr.Error(),
Duration: prototime.DurationToProto(time.Since(start)),
},
)
} else {
protolog.Info(
&ServerFinished{
Duration: prototime.DurationToProto(time.Since(start)),
},
)
}
}()
if port == 0 {
return ErrMustSpecifyPort
}
if registerFunc == nil {
return ErrMustSpecifyRegisterFunc
}
if opts.HTTPPort != 0 && opts.HTTPAddress != "" {
return ErrCannotSpecifyBothHTTPPortAndHTTPAddress
}
s := grpc.NewServer(grpc.MaxConcurrentStreams(math.MaxUint32))
registerFunc(s)
if opts.Version != nil {
protoversion.RegisterAPIServer(s, protoversion.NewAPIServer(opts.Version))
}
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
if err != nil {
return err
}
grpcErrC := make(chan error)
grpcDebugErrC := make(chan error)
httpErrC := make(chan error)
errCCount := 1
go func() { grpcErrC <- s.Serve(listener) }()
if opts.DebugPort != 0 {
errCCount++
debugServer := &graceful.Server{
Timeout: 1 * time.Second,
Server: &http.Server{
Addr: fmt.Sprintf(":%d", opts.DebugPort),
Handler: http.DefaultServeMux,
},
}
go func() { grpcDebugErrC <- debugServer.ListenAndServe() }()
}
if (opts.HTTPPort != 0 || opts.HTTPAddress != "") && (opts.Version != nil || opts.HTTPRegisterFunc != nil) {
time.Sleep(1 * time.Second)
ctx, cancel := context.WithCancel(context.Background())
var mux *runtime.ServeMux
if len(opts.ServeMuxOptions) == 0 {
mux = runtime.NewServeMux()
} else {
mux = runtime.NewServeMux(opts.ServeMuxOptions...)
}
conn, err := grpc.Dial(fmt.Sprintf("0.0.0.0:%d", port), grpc.WithInsecure())
if err != nil {
glog.Flush()
cancel()
return err
}
go func() {
<-ctx.Done()
_ = conn.Close()
}()
if opts.Version != nil {
if err := protoversion.RegisterAPIHandler(ctx, mux, conn); err != nil {
_ = conn.Close()
glog.Flush()
cancel()
return err
}
}
if opts.HTTPRegisterFunc != nil {
if err := opts.HTTPRegisterFunc(ctx, mux, conn); err != nil {
_ = conn.Close()
glog.Flush()
cancel()
return err
}
}
httpAddress := fmt.Sprintf(":%d", opts.HTTPPort)
if opts.HTTPAddress != "" {
httpAddress = opts.HTTPAddress
}
httpServer := &http.Server{
Addr: httpAddress,
Handler: mux,
}
gracefulServer := &graceful.Server{
Timeout: 1 * time.Second,
BeforeShutdown: opts.HTTPBeforeShutdown,
ShutdownInitiated: func() {
glog.Flush()
cancel()
if opts.HTTPShutdownInitiated != nil {
opts.HTTPShutdownInitiated()
}
},
Server: httpServer,
}
if opts.HTTPStart != nil {
close(opts.HTTPStart)
}
errCCount++
go func() {
if opts.HTTPListener != nil {
httpErrC <- gracefulServer.Serve(opts.HTTPListener)
} else {
httpErrC <- gracefulServer.ListenAndServe()
}
}()
}
protolog.Info(
&ServerStarted{
Port: uint32(port),
HttpPort: uint32(opts.HTTPPort),
DebugPort: uint32(opts.DebugPort),
HttpAddress: opts.HTTPAddress,
},
)
var errs []error
grpcStopped := false
for i := 0; i < errCCount; i++ {
select {
case grpcErr := <-grpcErrC:
if grpcErr != nil {
errs = append(errs, fmt.Errorf("grpc error: %s", grpcErr.Error()))
}
grpcStopped = true
case grpcDebugErr := <-grpcDebugErrC:
if grpcDebugErr != nil {
errs = append(errs, fmt.Errorf("grpc debug error: %s", grpcDebugErr.Error()))
}
if !grpcStopped {
s.Stop()
_ = listener.Close()
grpcStopped = true
}
case httpErr := <-httpErrC:
if httpErr != nil {
errs = append(errs, fmt.Errorf("http error: %s", httpErr.Error()))
}
if !grpcStopped {
s.Stop()
_ = listener.Close()
grpcStopped = true
}
}
}
if len(errs) > 0 {
return fmt.Errorf("%v", errs)
}
return nil
}
