Ejemplo n.º 1
0
func (l *logger) LogMethodInvocation(methodInvocation *MethodInvocation) {
	if methodInvocation.Error != "" {
		protolog.Error(methodInvocation)
	} else {
		protolog.Info(methodInvocation)
	}
}
Ejemplo n.º 2
0
func (a *discoveryAddresser) AssignRoles(cancel chan bool) (retErr error) {
	protolog.Info(&log.StartAssignRoles{})
	defer func() {
		protolog.Info(&log.FinishAssignRoles{errorToString(retErr)})
	}()
	var version int64
	oldServers := make(map[string]bool)
	oldRoles := make(map[string]*proto.ServerRole)
	oldMasters := make(map[uint64]string)
	oldReplicas := make(map[uint64][]string)
	var oldMinVersion int64
	// Reconstruct state from a previous run
	serverRoles, err := a.discoveryClient.GetAll(a.serverRoleDir())
	if err != nil {
		return err
	}
	for _, encodedServerRole := range serverRoles {
		serverRole, err := decodeServerRole(encodedServerRole)
		if err != nil {
			return err
		}
		if oldServerRole, ok := oldRoles[serverRole.Id]; !ok || oldServerRole.Version < serverRole.Version {
			oldRoles[serverRole.Id] = serverRole
			oldServers[serverRole.Id] = true
		}
		if version < serverRole.Version+1 {
			version = serverRole.Version + 1
		}
	}
	for _, oldServerRole := range oldRoles {
		for shard := range oldServerRole.Masters {
			oldMasters[shard] = oldServerRole.Id
		}
		for shard := range oldServerRole.Replicas {
			oldReplicas[shard] = append(oldReplicas[shard], oldServerRole.Id)
		}
	}
	err = a.discoveryClient.WatchAll(a.serverStateDir(), cancel,
		func(encodedServerStates map[string]string) error {
			if len(encodedServerStates) == 0 {
				return nil
			}
			newServerStates := make(map[string]*proto.ServerState)
			shardLocations := make(map[uint64][]string)
			newRoles := make(map[string]*proto.ServerRole)
			newMasters := make(map[uint64]string)
			newReplicas := make(map[uint64][]string)
			masterRolesPerServer := a.sharder.NumShards() / uint64(len(encodedServerStates))
			masterRolesRemainder := a.sharder.NumShards() % uint64(len(encodedServerStates))
			replicaRolesPerServer := (a.sharder.NumShards() * (a.sharder.NumReplicas())) / uint64(len(encodedServerStates))
			replicaRolesRemainder := (a.sharder.NumShards() * (a.sharder.NumReplicas())) % uint64(len(encodedServerStates))
			for _, encodedServerState := range encodedServerStates {
				serverState, err := decodeServerState(encodedServerState)
				if err != nil {
					return err
				}
				newServerStates[serverState.Id] = serverState
				newRoles[serverState.Id] = &proto.ServerRole{
					Id:       serverState.Id,
					Version:  version,
					Masters:  make(map[uint64]bool),
					Replicas: make(map[uint64]bool),
				}
				for shard := range serverState.Shards {
					shardLocations[shard] = append(shardLocations[shard], serverState.Id)
				}
			}
			// See if there's any roles we can delete
			minVersion := int64(math.MaxInt64)
			for _, serverState := range newServerStates {
				if serverState.Version < minVersion {
					minVersion = serverState.Version
				}
			}
			// Delete roles that no servers are using anymore
			if minVersion > oldMinVersion {
				oldMinVersion = minVersion
				serverRoles, err := a.discoveryClient.GetAll(a.serverRoleDir())
				if err != nil {
					return err
				}
				for key, encodedServerRole := range serverRoles {
					serverRole, err := decodeServerRole(encodedServerRole)
					if err != nil {
						return err
					}
					if serverRole.Version < minVersion {
						if err := a.discoveryClient.Delete(key); err != nil {
							return err
						}
						protolog.Info(&log.DeleteServerRole{serverRole})
					}
				}
			}
			// if the servers are identical to last time then we know we'll
			// assign shards the same way
			if sameServers(oldServers, newServerStates) {
				return nil
			}
		Master:
			for shard := uint64(0); shard < a.sharder.NumShards(); shard++ {
				if id, ok := oldMasters[shard]; ok {
					if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) {
						continue Master
					}
				}
				for _, id := range oldReplicas[shard] {
					if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) {
						continue Master
					}
				}
				for _, id := range shardLocations[shard] {
					if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) {
						continue Master
					}
				}
				for id := range newServerStates {
					if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) {
						continue Master
					}
				}
				protolog.Error(&log.FailedToAssignRoles{
					ServerStates: newServerStates,
					NumShards:    a.sharder.NumShards(),
					NumReplicas:  a.sharder.NumReplicas(),
				})
				return nil
			}
			for replica := uint64(0); replica < a.sharder.NumReplicas(); replica++ {
			Replica:
				for shard := uint64(0); shard < a.sharder.NumShards(); shard++ {
					if id, ok := oldMasters[shard]; ok {
						if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) {
							continue Replica
						}
					}
					for _, id := range oldReplicas[shard] {
						if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) {
							continue Replica
						}
					}
					for _, id := range shardLocations[shard] {
						if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) {
							continue Replica
						}
					}
					for id := range newServerStates {
						if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) {
							continue Replica
						}
					}
					for id := range newServerStates {
						if swapReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer) {
							continue Replica
						}
					}
					protolog.Error(&log.FailedToAssignRoles{
						ServerStates: newServerStates,
						NumShards:    a.sharder.NumShards(),
						NumReplicas:  a.sharder.NumReplicas(),
					})
					return nil
				}
			}
			addresses := proto.Addresses{
				Version:   version,
				Addresses: make(map[uint64]*proto.ShardAddresses),
			}
			for shard := uint64(0); shard < a.sharder.NumShards(); shard++ {
				addresses.Addresses[shard] = &proto.ShardAddresses{Replicas: make(map[string]bool)}
			}
			for id, serverRole := range newRoles {
				encodedServerRole, err := marshaler.MarshalToString(serverRole)
				if err != nil {
					return err
				}
				if err := a.discoveryClient.Set(a.serverRoleKeyVersion(id, version), encodedServerRole, 0); err != nil {
					return err
				}
				protolog.Info(&log.SetServerRole{serverRole})
				address := newServerStates[id].Address
				for shard := range serverRole.Masters {
					shardAddresses := addresses.Addresses[shard]
					shardAddresses.Master = address
					addresses.Addresses[shard] = shardAddresses
				}
				for shard := range serverRole.Replicas {
					shardAddresses := addresses.Addresses[shard]
					shardAddresses.Replicas[address] = true
					addresses.Addresses[shard] = shardAddresses
				}
			}
			encodedAddresses, err := marshaler.MarshalToString(&addresses)
			if err != nil {
				return err
			}
			if err := a.discoveryClient.Set(a.addressesKey(version), encodedAddresses, 0); err != nil {
				return err
			}
			protolog.Info(&log.SetAddresses{&addresses})
			version++
			oldServers = make(map[string]bool)
			for id := range newServerStates {
				oldServers[id] = true
			}
			oldRoles = newRoles
			oldMasters = newMasters
			oldReplicas = newReplicas
			return nil
		})
	if err == discovery.ErrCancelled {
		return ErrCancelled
	}
	return err
}
Ejemplo n.º 3
0
// Error logs an RPC call at the error level.
func Error(serviceName string, methodName string, request proto.Message, response proto.Message, err error, duration time.Duration) {
	protolog.Error(event(serviceName, methodName, request, response, err, duration))
}
Ejemplo n.º 4
0
// Serve serves stuff.
func Serve(
	port uint16,
	registerFunc func(*grpc.Server),
	opts ServeOptions,
) (retErr error) {
	start := time.Now()
	defer func() {
		if retErr != nil {
			protolog.Error(
				&ServerFinished{
					Error:    retErr.Error(),
					Duration: prototime.DurationToProto(time.Since(start)),
				},
			)
		} else {
			protolog.Info(
				&ServerFinished{
					Duration: prototime.DurationToProto(time.Since(start)),
				},
			)
		}
	}()
	if port == 0 {
		return ErrMustSpecifyPort
	}
	if registerFunc == nil {
		return ErrMustSpecifyRegisterFunc
	}
	if opts.HTTPPort != 0 && opts.HTTPAddress != "" {
		return ErrCannotSpecifyBothHTTPPortAndHTTPAddress
	}
	s := grpc.NewServer(grpc.MaxConcurrentStreams(math.MaxUint32))
	registerFunc(s)
	if opts.Version != nil {
		protoversion.RegisterAPIServer(s, protoversion.NewAPIServer(opts.Version))
	}
	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if err != nil {
		return err
	}
	grpcErrC := make(chan error)
	grpcDebugErrC := make(chan error)
	httpErrC := make(chan error)
	errCCount := 1
	go func() { grpcErrC <- s.Serve(listener) }()
	if opts.DebugPort != 0 {
		errCCount++
		debugServer := &graceful.Server{
			Timeout: 1 * time.Second,
			Server: &http.Server{
				Addr:    fmt.Sprintf(":%d", opts.DebugPort),
				Handler: http.DefaultServeMux,
			},
		}
		go func() { grpcDebugErrC <- debugServer.ListenAndServe() }()
	}
	if (opts.HTTPPort != 0 || opts.HTTPAddress != "") && (opts.Version != nil || opts.HTTPRegisterFunc != nil) {
		time.Sleep(1 * time.Second)
		ctx, cancel := context.WithCancel(context.Background())
		var mux *runtime.ServeMux
		if len(opts.ServeMuxOptions) == 0 {
			mux = runtime.NewServeMux()
		} else {
			mux = runtime.NewServeMux(opts.ServeMuxOptions...)
		}
		conn, err := grpc.Dial(fmt.Sprintf("0.0.0.0:%d", port), grpc.WithInsecure())
		if err != nil {
			glog.Flush()
			cancel()
			return err
		}
		go func() {
			<-ctx.Done()
			_ = conn.Close()
		}()
		if opts.Version != nil {
			if err := protoversion.RegisterAPIHandler(ctx, mux, conn); err != nil {
				_ = conn.Close()
				glog.Flush()
				cancel()
				return err
			}
		}
		if opts.HTTPRegisterFunc != nil {
			if err := opts.HTTPRegisterFunc(ctx, mux, conn); err != nil {
				_ = conn.Close()
				glog.Flush()
				cancel()
				return err
			}
		}
		httpAddress := fmt.Sprintf(":%d", opts.HTTPPort)
		if opts.HTTPAddress != "" {
			httpAddress = opts.HTTPAddress
		}
		httpServer := &http.Server{
			Addr:    httpAddress,
			Handler: mux,
		}
		gracefulServer := &graceful.Server{
			Timeout:        1 * time.Second,
			BeforeShutdown: opts.HTTPBeforeShutdown,
			ShutdownInitiated: func() {
				glog.Flush()
				cancel()
				if opts.HTTPShutdownInitiated != nil {
					opts.HTTPShutdownInitiated()
				}
			},
			Server: httpServer,
		}
		if opts.HTTPStart != nil {
			close(opts.HTTPStart)
		}
		errCCount++
		go func() {
			if opts.HTTPListener != nil {
				httpErrC <- gracefulServer.Serve(opts.HTTPListener)
			} else {
				httpErrC <- gracefulServer.ListenAndServe()
			}
		}()
	}
	protolog.Info(
		&ServerStarted{
			Port:        uint32(port),
			HttpPort:    uint32(opts.HTTPPort),
			DebugPort:   uint32(opts.DebugPort),
			HttpAddress: opts.HTTPAddress,
		},
	)
	var errs []error
	grpcStopped := false
	for i := 0; i < errCCount; i++ {
		select {
		case grpcErr := <-grpcErrC:
			if grpcErr != nil {
				errs = append(errs, fmt.Errorf("grpc error: %s", grpcErr.Error()))
			}
			grpcStopped = true
		case grpcDebugErr := <-grpcDebugErrC:
			if grpcDebugErr != nil {
				errs = append(errs, fmt.Errorf("grpc debug error: %s", grpcDebugErr.Error()))
			}
			if !grpcStopped {
				s.Stop()
				_ = listener.Close()
				grpcStopped = true
			}
		case httpErr := <-httpErrC:
			if httpErr != nil {
				errs = append(errs, fmt.Errorf("http error: %s", httpErr.Error()))
			}
			if !grpcStopped {
				s.Stop()
				_ = listener.Close()
				grpcStopped = true
			}
		}
	}
	if len(errs) > 0 {
		return fmt.Errorf("%v", errs)
	}
	return nil
}