Example #1
0
func (proxy *LeverProxy) handleInStream(stream *http2stream.HTTP2Stream) {
	headers := stream.GetHeaders()
	err := expectHeaders(
		headers,
		"lever-url",
		"x-lever-src-env",
		"x-lever-dest-instance-id",
		"x-lever-dest-container-id",
		"x-lever-serving-id",
		"x-lever-code-version",
		"x-lever-inst-resource-id",
		"x-lever-inst-session-id",
		"x-lever-res-resource-id",
		"x-lever-res-session-id",
	)
	if err != nil {
		proxy.inLogger.WithFields("err", err).Error("")
		stream.Write(&http2stream.MsgError{Err: err})
		return
	}

	leverURL, err := core.ParseLeverURL(headers["lever-url"][0])
	if err != nil {
		proxy.inLogger.WithFields(
			"err", err, "leverURL", headers["lever-url"][0]).Error(
			"Unable to parse Lever URL")
	}

	if !core.IsInternalEnvironment(leverURL.Environment) {
		err = fmt.Errorf("Cannot route to dest env")
		proxy.inLogger.WithFields(
			"err", err,
			"leverEnv", leverURL.Environment,
		).Error("")
		stream.Write(&http2stream.MsgError{Err: err})
		return
	}

	srcEnv := headers["x-lever-src-env"][0]
	instanceID := headers["x-lever-dest-instance-id"][0]
	containerID := headers["x-lever-dest-container-id"][0]
	servingID := headers["x-lever-serving-id"][0]
	codeVersionInt, err := strconv.Atoi(headers["x-lever-code-version"][0])
	if err != nil {
		proxy.inLogger.WithFields("err", err).Error("Cannot parse code version")
		stream.Write(&http2stream.MsgError{Err: err})
		return
	}
	codeVersion := int64(codeVersionInt)
	levInstResourceID := headers["x-lever-inst-resource-id"][0]
	levInstSessionID := headers["x-lever-inst-session-id"][0]
	levResResourceID := headers["x-lever-res-resource-id"][0]
	levResSessionID := headers["x-lever-res-session-id"][0]

	if instanceID == "" {
		instanceID, err = proxy.manager.RandomInstaceID(servingID)
		if err != nil {
			proxy.inLogger.WithFields(
				"err", err,
				"leverEnv", leverURL.Environment,
			).Error("Could not find an instanceID for provided servingID")
			stream.Write(&http2stream.MsgError{Err: err})
			return
		}
	}

	streamID := leverutil.RandomID()
	proxy.inLogger.WithFields(
		"leverURL", leverURL.String(),
		"srcEnv", srcEnv,
		"leverInstanceID", instanceID,
		"containerID", containerID,
		"servingID", servingID,
		"levInstSessionID", levInstSessionID,
		"levInstResourceID", levInstResourceID,
		"streamID", streamID,
	).Debug("Receiving stream")
	streamLogger := proxy.inLogger.WithFields("streamID", streamID)

	if !core.IsInternalEnvironment(leverURL.Environment) {
		err = fmt.Errorf("Environment not routable internally")
		streamLogger.WithFields(
			"err", err,
			"leverEnv", leverURL.Environment,
		).Error("")
		stream.Write(&http2stream.MsgError{Err: err})
		return
	}

	_, ownIP, instanceAddr, keepAliveFun, err :=
		proxy.manager.EnsureInfrastructureInitialized(&hostman.InstanceInfo{
			Environment:       leverURL.Environment,
			Service:           leverURL.Service,
			InstanceID:        instanceID,
			ContainerID:       containerID,
			ServingID:         servingID,
			LevInstResourceID: levInstResourceID,
			LevInstSessionID:  levInstSessionID,
		})
	if err != nil {
		streamLogger.WithFields("err", err).Error(
			"Error initializing instance")
		stream.Write(&http2stream.MsgError{Err: err})
		return
	}

	err = proxy.serveOut(leverURL.Environment, ownIP)
	if err != nil {
		streamLogger.WithFields("err", err).Error(
			"Error listening on env network")
		stream.Write(&http2stream.MsgError{Err: err})
		return
	}

	destStreamI, err := leverutil.ExpBackoff(
		func() (clientStream interface{}, err error, finalErr error) {
			clientStream, err = proxy.client.NewStream(instanceAddr)
			if err != nil {
				if strings.Contains(err.Error(), "connection refused") &&
					proxy.manager.IsInstanceAlive(servingID, instanceID) {
					// Retry.
					return nil, err, nil
				}
				if err == leverutil.ErrNotYetConstructed ||
					err == leverutil.ErrWasDestructed {
					// Retry.
					return nil, err, nil
				}
				return nil, nil, err
			}
			return clientStream, nil, nil
		}, 10*time.Millisecond, 15*time.Second)
	if err != nil {
		streamLogger.WithFields(
			"err", err,
			"instanceAddr", instanceAddr,
		).Error("Error trying to create client stream to dest")
		stream.Write(&http2stream.MsgError{Err: err})
		return
	}
	destStream := destStreamI.(*http2stream.HTTP2Stream)

	addHeaders := make(map[string][]string)
	if srcEnv != "" {
		addHeaders["x-lever-src-env"] = []string{srcEnv}
	}
	addHeaders["x-lever-internal-rpc-gateway"] = []string{
		ownIP + ":" + EnvOutListenPortFlag.Get()}

	startTime := time.Now()
	firstHeaders := true
	stream.ProxyTo(
		destStream,
		func(msg http2stream.MsgItem) []http2stream.MsgItem {
			proxy.client.KeepAlive(instanceAddr)
			keepAliveFun(
				leverURL.Resource, levResResourceID, levResSessionID)
			return proxy.filterTo(&firstHeaders, addHeaders, msg)
		},
		func(msg http2stream.MsgItem) []http2stream.MsgItem {
			proxy.client.KeepAlive(instanceAddr)
			keepAliveFun(
				leverURL.Resource, levResResourceID, levResSessionID)
			return noFilter(msg)
		})
	// Wait for RPC to finish.
	<-destStream.Closed()
	rpcNanos := uint64(time.Now().Sub(startTime).Nanoseconds())
	streamLogger.WithFields("rpcNanos", rpcNanos).Debug("RPC nanos")

	// Send RPC stats to fleettracker.
	// TODO: For services with high load, we should not send info on every
	//       RPC. They should be batched and sent say... every ~50ms
	//       (well below tracker tick interval).
	err = fleettracker.OnRPC(proxy.grpcPool, &fleettracker.RPCEvent{
		Environment: leverURL.Environment,
		Service:     leverURL.Service,
		ServingID:   servingID,
		CodeVersion: codeVersion,
		IsAdmin:     core.IsAdmin(leverURL),
		RpcNanos:    rpcNanos,
	})
	if err != nil {
		streamLogger.WithFields("err", err).Error(
			"Failed to send RPC stats to fleettracker")
	}
}
Example #2
0
func (client *Client) invokeInternal(
	replyObj interface{}, leverURL *core.LeverURL,
	args ...interface{}) (err error) {
	if IsChanMethod(leverURL.Method) {
		return fmt.Errorf(
			"Use InvokeChan / InvokeChanURL for streaming methods")
	}
	if leverURL.Environment == "" && OwnEnvironment == "" {
		return fmt.Errorf(
			"Environment not specified and cannot be deduced")
	}
	if leverURL.Environment == "" {
		leverURL.Environment = OwnEnvironment
	}

	var dialTo string
	if client.ForceHost != "" {
		dialTo = client.ForceHost
	} else {
		if (core.IsInternalEnvironment(leverURL.Environment) ||
			leverURL.Environment == OwnEnvironment) &&
			internalRPCGateway != "" {
			dialTo = internalRPCGateway
		} else {
			dialTo = leverURL.Environment
		}
	}
	conn, err := client.conns.Dial(dialTo)
	if err != nil {
		return err
	}

	rpc := &core.RPC{}
	if len(args) == 1 {
		byteArgs, ok := args[0].([]byte)
		if ok {
			// Byte args case.
			rpc.ArgsOneof = &core.RPC_ByteArgs{
				ByteArgs: byteArgs,
			}
		}
	}
	if rpc.ArgsOneof == nil {
		// Non-byte args case.
		var encdArgs *core.JSONArray
		encdArgs, err = encodeArgs(args)
		if err != nil {
			return err
		}
		rpc.ArgsOneof = &core.RPC_Args{
			Args: encdArgs,
		}
	}
	reply, err := core.SendLeverRPC(
		conn, context.Background(), leverURL, rpc)
	if err != nil {
		return err
	}

	if reply.GetResultOneof() == nil {
		return fmt.Errorf("Received nil result oneof")
	}
	switch result := reply.GetResultOneof().(type) {
	case *core.RPCReply_ByteResult:
		byteReplyObj, ok := replyObj.(*[]byte)
		if !ok {
			return fmt.Errorf(
				"replyObj needs to be type *[]byte when RPC returns bytes")
		}
		*byteReplyObj = result.ByteResult
		return nil
	case *core.RPCReply_ByteError:
		return &RemoteByteError{Err: result.ByteError}
	case *core.RPCReply_Result:
		return decodeArgAsValue(result.Result, replyObj)
	case *core.RPCReply_Error:
		return &RemoteError{Err: buildDataForJSON(result.Error)}
	default:
		return fmt.Errorf("Invalid type")
	}
}
Example #3
0
func (client *Client) invokeChanInternal(
	leverURL *core.LeverURL, args ...interface{}) (stream Stream, err error) {
	if !IsChanMethod(leverURL.Method) {
		return nil, fmt.Errorf(
			"Use Invoke / InvokeURL for non-streaming methods")
	}
	if leverURL.Environment == "" && OwnEnvironment == "" {
		return nil, fmt.Errorf(
			"Environment not specified and cannot be deduced")
	}
	if leverURL.Environment == "" {
		leverURL.Environment = OwnEnvironment
	}

	var dialTo string
	if client.ForceHost != "" {
		dialTo = client.ForceHost
	} else {
		if (core.IsInternalEnvironment(leverURL.Environment) ||
			leverURL.Environment == OwnEnvironment) &&
			internalRPCGateway != "" {
			dialTo = internalRPCGateway
		} else {
			dialTo = leverURL.Environment
		}
	}
	conn, err := client.conns.Dial(dialTo)
	if err != nil {
		return nil, err
	}

	rpc := &core.RPC{}
	if len(args) == 1 {
		byteArgs, ok := args[0].([]byte)
		if ok {
			// Byte args case.
			rpc.ArgsOneof = &core.RPC_ByteArgs{
				ByteArgs: byteArgs,
			}
		}
	}
	if rpc.ArgsOneof == nil {
		// Non-byte args case.
		var encdArgs *core.JSONArray
		encdArgs, err = encodeArgs(args)
		if err != nil {
			return nil, err
		}
		rpc.ArgsOneof = &core.RPC_Args{
			Args: encdArgs,
		}
	}
	grpcStream, err := core.SendStreamingLeverRPC(
		conn, context.Background(), leverURL)
	if err != nil {
		return nil, err
	}

	err = grpcStream.Send(&core.StreamMessage{
		MessageOneof: &core.StreamMessage_Rpc{
			Rpc: rpc,
		},
	})
	if err != nil {
		grpcStream.CloseSend()
		return nil, err
	}
	// First message received must be empty.
	fstMsg, err := grpcStream.Recv()
	if err != nil {
		return nil, err
	}
	if fstMsg.GetMessageOneof() != nil {
		return nil, fmt.Errorf("First server message needs to be empty")
	}
	return newClientStream(grpcStream), nil
}
Example #4
0
func (proxy *LeverProxy) handleExtOutStream(
	srcEnv string, stream *http2stream.HTTP2Stream, isExtStream bool) {
	streamLogger := proxy.outLogger
	if isExtStream {
		streamLogger = proxy.extLogger
	}

	headers := stream.GetHeaders()
	err := expectHeaders(headers, "lever-url")
	if err != nil {
		streamLogger.WithFields("err", err).Error("")
		stream.Write(&http2stream.MsgError{Err: err})
		return
	}

	leverURL, err := core.ParseLeverURL(headers["lever-url"][0])
	if err != nil {
		proxy.inLogger.WithFields(
			"err", err, "leverURL", headers["lever-url"][0]).Error(
			"Unable to parse Lever URL")
	}

	if !core.IsInternalEnvironment(leverURL.Environment) {
		err = fmt.Errorf("Cannot route to dest env")
		streamLogger.WithFields(
			"err", err,
			"leverEnv", leverURL.Environment,
		).Error("")
		stream.Write(&http2stream.MsgError{Err: err})
		return
	}

	streamID := leverutil.RandomID()
	streamLogger.WithFields(
		"leverURL", leverURL.String(),
		"srcEnv", srcEnv,
		"streamID", streamID,
	).Debug("Receiving stream")
	streamLogger = streamLogger.WithFields("streamID", streamID)

	isFromExt := !core.IsInternalEnvironment(srcEnv)
	hostInfo, err := proxy.finder.GetHost(
		leverURL.Environment, leverURL.Service, leverURL.Resource, isFromExt)
	if err != nil {
		streamLogger.WithFields(
			"err", err,
			"leverURL", leverURL.String(),
		).Error("Error trying to find / allocate host for request")
		stream.Write(&http2stream.MsgError{Err: err})
		return
	}

	streamLogger.WithFields(
		"leverInstanceID", hostInfo.InstanceID,
		"containerID", hostInfo.ContainerID,
		"servingID", hostInfo.ServingID,
		"hostProxyAddr", hostInfo.HostAddr,
		"isNewInstance", hostInfo.IsNewInstance,
	).Debug("Routing to host proxy")
	destStream, err := proxy.client.NewStream(hostInfo.HostAddr)
	if err != nil {
		streamLogger.WithFields(
			"err", err,
			"hostProxyAddr", hostInfo.HostAddr,
		).Error("Error trying to create client stream to dest")
		stream.Write(&http2stream.MsgError{Err: err})
		return
	}

	addHeaders := make(map[string][]string)
	addHeaders["x-lever-src-env"] = []string{srcEnv}
	addHeaders["x-lever-dest-instance-id"] = []string{hostInfo.InstanceID}
	addHeaders["x-lever-dest-container-id"] = []string{hostInfo.ContainerID}
	addHeaders["x-lever-serving-id"] = []string{hostInfo.ServingID}
	addHeaders["x-lever-code-version"] = []string{
		strconv.Itoa(int(hostInfo.CodeVersion))}
	addHeaders["x-lever-inst-resource-id"] = []string{
		hostInfo.LevInstResourceID}
	addHeaders["x-lever-inst-session-id"] = []string{hostInfo.LevInstSessionID}
	addHeaders["x-lever-res-resource-id"] = []string{hostInfo.LevResResourceID}
	addHeaders["x-lever-res-session-id"] = []string{hostInfo.LevResSessionID}

	firstHeaders := true
	stream.ProxyTo(
		destStream,
		func(msg http2stream.MsgItem) []http2stream.MsgItem {
			proxy.client.KeepAlive(hostInfo.HostAddr)
			return proxy.filterTo(&firstHeaders, addHeaders, msg)
		},
		func(msg http2stream.MsgItem) []http2stream.MsgItem {
			proxy.client.KeepAlive(hostInfo.HostAddr)
			return noFilter(msg)
		})
}