func (l *logger) LogMethodInvocation(methodInvocation *MethodInvocation) { if methodInvocation.Error != "" { protolog.Error(methodInvocation) } else { protolog.Info(methodInvocation) } }
func (a *discoveryAddresser) AssignRoles(cancel chan bool) (retErr error) { protolog.Info(&log.StartAssignRoles{}) defer func() { protolog.Info(&log.FinishAssignRoles{errorToString(retErr)}) }() var version int64 oldServers := make(map[string]bool) oldRoles := make(map[string]*proto.ServerRole) oldMasters := make(map[uint64]string) oldReplicas := make(map[uint64][]string) var oldMinVersion int64 // Reconstruct state from a previous run serverRoles, err := a.discoveryClient.GetAll(a.serverRoleDir()) if err != nil { return err } for _, encodedServerRole := range serverRoles { serverRole, err := decodeServerRole(encodedServerRole) if err != nil { return err } if oldServerRole, ok := oldRoles[serverRole.Id]; !ok || oldServerRole.Version < serverRole.Version { oldRoles[serverRole.Id] = serverRole oldServers[serverRole.Id] = true } if version < serverRole.Version+1 { version = serverRole.Version + 1 } } for _, oldServerRole := range oldRoles { for shard := range oldServerRole.Masters { oldMasters[shard] = oldServerRole.Id } for shard := range oldServerRole.Replicas { oldReplicas[shard] = append(oldReplicas[shard], oldServerRole.Id) } } err = a.discoveryClient.WatchAll(a.serverStateDir(), cancel, func(encodedServerStates map[string]string) error { if len(encodedServerStates) == 0 { return nil } newServerStates := make(map[string]*proto.ServerState) shardLocations := make(map[uint64][]string) newRoles := make(map[string]*proto.ServerRole) newMasters := make(map[uint64]string) newReplicas := make(map[uint64][]string) masterRolesPerServer := a.sharder.NumShards() / uint64(len(encodedServerStates)) masterRolesRemainder := a.sharder.NumShards() % uint64(len(encodedServerStates)) replicaRolesPerServer := (a.sharder.NumShards() * (a.sharder.NumReplicas())) / uint64(len(encodedServerStates)) replicaRolesRemainder := (a.sharder.NumShards() * (a.sharder.NumReplicas())) % uint64(len(encodedServerStates)) for _, encodedServerState := range encodedServerStates { serverState, err := decodeServerState(encodedServerState) if err != nil { return err } newServerStates[serverState.Id] = serverState newRoles[serverState.Id] = &proto.ServerRole{ Id: serverState.Id, Version: version, Masters: make(map[uint64]bool), Replicas: make(map[uint64]bool), } for shard := range serverState.Shards { shardLocations[shard] = append(shardLocations[shard], serverState.Id) } } // See if there's any roles we can delete minVersion := int64(math.MaxInt64) for _, serverState := range newServerStates { if serverState.Version < minVersion { minVersion = serverState.Version } } // Delete roles that no servers are using anymore if minVersion > oldMinVersion { oldMinVersion = minVersion serverRoles, err := a.discoveryClient.GetAll(a.serverRoleDir()) if err != nil { return err } for key, encodedServerRole := range serverRoles { serverRole, err := decodeServerRole(encodedServerRole) if err != nil { return err } if serverRole.Version < minVersion { if err := a.discoveryClient.Delete(key); err != nil { return err } protolog.Info(&log.DeleteServerRole{serverRole}) } } } // if the servers are identical to last time then we know we'll // assign shards the same way if sameServers(oldServers, newServerStates) { return nil } Master: for shard := uint64(0); shard < a.sharder.NumShards(); shard++ { if id, ok := oldMasters[shard]; ok { if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) { continue Master } } for _, id := range oldReplicas[shard] { if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) { continue Master } } for _, id := range shardLocations[shard] { if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) { continue Master } } for id := range newServerStates { if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) { continue Master } } protolog.Error(&log.FailedToAssignRoles{ ServerStates: newServerStates, NumShards: a.sharder.NumShards(), NumReplicas: a.sharder.NumReplicas(), }) return nil } for replica := uint64(0); replica < a.sharder.NumReplicas(); replica++ { Replica: for shard := uint64(0); shard < a.sharder.NumShards(); shard++ { if id, ok := oldMasters[shard]; ok { if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) { continue Replica } } for _, id := range oldReplicas[shard] { if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) { continue Replica } } for _, id := range shardLocations[shard] { if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) { continue Replica } } for id := range newServerStates { if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) { continue Replica } } for id := range newServerStates { if swapReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer) { continue Replica } } protolog.Error(&log.FailedToAssignRoles{ ServerStates: newServerStates, NumShards: a.sharder.NumShards(), NumReplicas: a.sharder.NumReplicas(), }) return nil } } addresses := proto.Addresses{ Version: version, Addresses: make(map[uint64]*proto.ShardAddresses), } for shard := uint64(0); shard < a.sharder.NumShards(); shard++ { addresses.Addresses[shard] = &proto.ShardAddresses{Replicas: make(map[string]bool)} } for id, serverRole := range newRoles { encodedServerRole, err := marshaler.MarshalToString(serverRole) if err != nil { return err } if err := a.discoveryClient.Set(a.serverRoleKeyVersion(id, version), encodedServerRole, 0); err != nil { return err } protolog.Info(&log.SetServerRole{serverRole}) address := newServerStates[id].Address for shard := range serverRole.Masters { shardAddresses := addresses.Addresses[shard] shardAddresses.Master = address addresses.Addresses[shard] = shardAddresses } for shard := range serverRole.Replicas { shardAddresses := addresses.Addresses[shard] shardAddresses.Replicas[address] = true addresses.Addresses[shard] = shardAddresses } } encodedAddresses, err := marshaler.MarshalToString(&addresses) if err != nil { return err } if err := a.discoveryClient.Set(a.addressesKey(version), encodedAddresses, 0); err != nil { return err } protolog.Info(&log.SetAddresses{&addresses}) version++ oldServers = make(map[string]bool) for id := range newServerStates { oldServers[id] = true } oldRoles = newRoles oldMasters = newMasters oldReplicas = newReplicas return nil }) if err == discovery.ErrCancelled { return ErrCancelled } return err }
// Error logs an RPC call at the error level. func Error(serviceName string, methodName string, request proto.Message, response proto.Message, err error, duration time.Duration) { protolog.Error(event(serviceName, methodName, request, response, err, duration)) }
// Serve serves stuff. func Serve( port uint16, registerFunc func(*grpc.Server), opts ServeOptions, ) (retErr error) { start := time.Now() defer func() { if retErr != nil { protolog.Error( &ServerFinished{ Error: retErr.Error(), Duration: prototime.DurationToProto(time.Since(start)), }, ) } else { protolog.Info( &ServerFinished{ Duration: prototime.DurationToProto(time.Since(start)), }, ) } }() if port == 0 { return ErrMustSpecifyPort } if registerFunc == nil { return ErrMustSpecifyRegisterFunc } if opts.HTTPPort != 0 && opts.HTTPAddress != "" { return ErrCannotSpecifyBothHTTPPortAndHTTPAddress } s := grpc.NewServer(grpc.MaxConcurrentStreams(math.MaxUint32)) registerFunc(s) if opts.Version != nil { protoversion.RegisterAPIServer(s, protoversion.NewAPIServer(opts.Version)) } listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) if err != nil { return err } grpcErrC := make(chan error) grpcDebugErrC := make(chan error) httpErrC := make(chan error) errCCount := 1 go func() { grpcErrC <- s.Serve(listener) }() if opts.DebugPort != 0 { errCCount++ debugServer := &graceful.Server{ Timeout: 1 * time.Second, Server: &http.Server{ Addr: fmt.Sprintf(":%d", opts.DebugPort), Handler: http.DefaultServeMux, }, } go func() { grpcDebugErrC <- debugServer.ListenAndServe() }() } if (opts.HTTPPort != 0 || opts.HTTPAddress != "") && (opts.Version != nil || opts.HTTPRegisterFunc != nil) { time.Sleep(1 * time.Second) ctx, cancel := context.WithCancel(context.Background()) var mux *runtime.ServeMux if len(opts.ServeMuxOptions) == 0 { mux = runtime.NewServeMux() } else { mux = runtime.NewServeMux(opts.ServeMuxOptions...) } conn, err := grpc.Dial(fmt.Sprintf("0.0.0.0:%d", port), grpc.WithInsecure()) if err != nil { glog.Flush() cancel() return err } go func() { <-ctx.Done() _ = conn.Close() }() if opts.Version != nil { if err := protoversion.RegisterAPIHandler(ctx, mux, conn); err != nil { _ = conn.Close() glog.Flush() cancel() return err } } if opts.HTTPRegisterFunc != nil { if err := opts.HTTPRegisterFunc(ctx, mux, conn); err != nil { _ = conn.Close() glog.Flush() cancel() return err } } httpAddress := fmt.Sprintf(":%d", opts.HTTPPort) if opts.HTTPAddress != "" { httpAddress = opts.HTTPAddress } httpServer := &http.Server{ Addr: httpAddress, Handler: mux, } gracefulServer := &graceful.Server{ Timeout: 1 * time.Second, BeforeShutdown: opts.HTTPBeforeShutdown, ShutdownInitiated: func() { glog.Flush() cancel() if opts.HTTPShutdownInitiated != nil { opts.HTTPShutdownInitiated() } }, Server: httpServer, } if opts.HTTPStart != nil { close(opts.HTTPStart) } errCCount++ go func() { if opts.HTTPListener != nil { httpErrC <- gracefulServer.Serve(opts.HTTPListener) } else { httpErrC <- gracefulServer.ListenAndServe() } }() } protolog.Info( &ServerStarted{ Port: uint32(port), HttpPort: uint32(opts.HTTPPort), DebugPort: uint32(opts.DebugPort), HttpAddress: opts.HTTPAddress, }, ) var errs []error grpcStopped := false for i := 0; i < errCCount; i++ { select { case grpcErr := <-grpcErrC: if grpcErr != nil { errs = append(errs, fmt.Errorf("grpc error: %s", grpcErr.Error())) } grpcStopped = true case grpcDebugErr := <-grpcDebugErrC: if grpcDebugErr != nil { errs = append(errs, fmt.Errorf("grpc debug error: %s", grpcDebugErr.Error())) } if !grpcStopped { s.Stop() _ = listener.Close() grpcStopped = true } case httpErr := <-httpErrC: if httpErr != nil { errs = append(errs, fmt.Errorf("http error: %s", httpErr.Error())) } if !grpcStopped { s.Stop() _ = listener.Close() grpcStopped = true } } } if len(errs) > 0 { return fmt.Errorf("%v", errs) } return nil }