func (proxy *LeverProxy) handleInStream(stream *http2stream.HTTP2Stream) {
headers := stream.GetHeaders()
err := expectHeaders(
headers,
"lever-url",
"x-lever-src-env",
"x-lever-dest-instance-id",
"x-lever-dest-container-id",
"x-lever-serving-id",
"x-lever-code-version",
"x-lever-inst-resource-id",
"x-lever-inst-session-id",
"x-lever-res-resource-id",
"x-lever-res-session-id",
)
if err != nil {
proxy.inLogger.WithFields("err", err).Error("")
stream.Write(&http2stream.MsgError{Err: err})
return
}
leverURL, err := core.ParseLeverURL(headers["lever-url"][0])
if err != nil {
proxy.inLogger.WithFields(
"err", err, "leverURL", headers["lever-url"][0]).Error(
"Unable to parse Lever URL")
}
if !core.IsInternalEnvironment(leverURL.Environment) {
err = fmt.Errorf("Cannot route to dest env")
proxy.inLogger.WithFields(
"err", err,
"leverEnv", leverURL.Environment,
).Error("")
stream.Write(&http2stream.MsgError{Err: err})
return
}
srcEnv := headers["x-lever-src-env"][0]
instanceID := headers["x-lever-dest-instance-id"][0]
containerID := headers["x-lever-dest-container-id"][0]
servingID := headers["x-lever-serving-id"][0]
codeVersionInt, err := strconv.Atoi(headers["x-lever-code-version"][0])
if err != nil {
proxy.inLogger.WithFields("err", err).Error("Cannot parse code version")
stream.Write(&http2stream.MsgError{Err: err})
return
}
codeVersion := int64(codeVersionInt)
levInstResourceID := headers["x-lever-inst-resource-id"][0]
levInstSessionID := headers["x-lever-inst-session-id"][0]
levResResourceID := headers["x-lever-res-resource-id"][0]
levResSessionID := headers["x-lever-res-session-id"][0]
if instanceID == "" {
instanceID, err = proxy.manager.RandomInstaceID(servingID)
if err != nil {
proxy.inLogger.WithFields(
"err", err,
"leverEnv", leverURL.Environment,
).Error("Could not find an instanceID for provided servingID")
stream.Write(&http2stream.MsgError{Err: err})
return
}
}
streamID := leverutil.RandomID()
proxy.inLogger.WithFields(
"leverURL", leverURL.String(),
"srcEnv", srcEnv,
"leverInstanceID", instanceID,
"containerID", containerID,
"servingID", servingID,
"levInstSessionID", levInstSessionID,
"levInstResourceID", levInstResourceID,
"streamID", streamID,
).Debug("Receiving stream")
streamLogger := proxy.inLogger.WithFields("streamID", streamID)
if !core.IsInternalEnvironment(leverURL.Environment) {
err = fmt.Errorf("Environment not routable internally")
streamLogger.WithFields(
"err", err,
"leverEnv", leverURL.Environment,
).Error("")
stream.Write(&http2stream.MsgError{Err: err})
return
}
_, ownIP, instanceAddr, keepAliveFun, err :=
proxy.manager.EnsureInfrastructureInitialized(&hostman.InstanceInfo{
Environment: leverURL.Environment,
Service: leverURL.Service,
InstanceID: instanceID,
ContainerID: containerID,
ServingID: servingID,
LevInstResourceID: levInstResourceID,
LevInstSessionID: levInstSessionID,
})
if err != nil {
streamLogger.WithFields("err", err).Error(
"Error initializing instance")
stream.Write(&http2stream.MsgError{Err: err})
return
}
err = proxy.serveOut(leverURL.Environment, ownIP)
if err != nil {
streamLogger.WithFields("err", err).Error(
"Error listening on env network")
stream.Write(&http2stream.MsgError{Err: err})
return
}
destStreamI, err := leverutil.ExpBackoff(
func() (clientStream interface{}, err error, finalErr error) {
clientStream, err = proxy.client.NewStream(instanceAddr)
if err != nil {
if strings.Contains(err.Error(), "connection refused") &&
proxy.manager.IsInstanceAlive(servingID, instanceID) {
// Retry.
return nil, err, nil
}
if err == leverutil.ErrNotYetConstructed ||
err == leverutil.ErrWasDestructed {
// Retry.
return nil, err, nil
}
return nil, nil, err
}
return clientStream, nil, nil
}, 10*time.Millisecond, 15*time.Second)
if err != nil {
streamLogger.WithFields(
"err", err,
"instanceAddr", instanceAddr,
).Error("Error trying to create client stream to dest")
stream.Write(&http2stream.MsgError{Err: err})
return
}
destStream := destStreamI.(*http2stream.HTTP2Stream)
addHeaders := make(map[string][]string)
if srcEnv != "" {
addHeaders["x-lever-src-env"] = []string{srcEnv}
}
addHeaders["x-lever-internal-rpc-gateway"] = []string{
ownIP + ":" + EnvOutListenPortFlag.Get()}
startTime := time.Now()
firstHeaders := true
stream.ProxyTo(
destStream,
func(msg http2stream.MsgItem) []http2stream.MsgItem {
proxy.client.KeepAlive(instanceAddr)
keepAliveFun(
leverURL.Resource, levResResourceID, levResSessionID)
return proxy.filterTo(&firstHeaders, addHeaders, msg)
},
func(msg http2stream.MsgItem) []http2stream.MsgItem {
proxy.client.KeepAlive(instanceAddr)
keepAliveFun(
leverURL.Resource, levResResourceID, levResSessionID)
return noFilter(msg)
})
// Wait for RPC to finish.
<-destStream.Closed()
rpcNanos := uint64(time.Now().Sub(startTime).Nanoseconds())
streamLogger.WithFields("rpcNanos", rpcNanos).Debug("RPC nanos")
// Send RPC stats to fleettracker.
// TODO: For services with high load, we should not send info on every
// RPC. They should be batched and sent say... every ~50ms
// (well below tracker tick interval).
err = fleettracker.OnRPC(proxy.grpcPool, &fleettracker.RPCEvent{
Environment: leverURL.Environment,
Service: leverURL.Service,
ServingID: servingID,
CodeVersion: codeVersion,
IsAdmin: core.IsAdmin(leverURL),
RpcNanos: rpcNanos,
})
if err != nil {
streamLogger.WithFields("err", err).Error(
"Failed to send RPC stats to fleettracker")
}
}
func (client *Client) invokeInternal(
replyObj interface{}, leverURL *core.LeverURL,
args ...interface{}) (err error) {
if IsChanMethod(leverURL.Method) {
return fmt.Errorf(
"Use InvokeChan / InvokeChanURL for streaming methods")
}
if leverURL.Environment == "" && OwnEnvironment == "" {
return fmt.Errorf(
"Environment not specified and cannot be deduced")
}
if leverURL.Environment == "" {
leverURL.Environment = OwnEnvironment
}
var dialTo string
if client.ForceHost != "" {
dialTo = client.ForceHost
} else {
if (core.IsInternalEnvironment(leverURL.Environment) ||
leverURL.Environment == OwnEnvironment) &&
internalRPCGateway != "" {
dialTo = internalRPCGateway
} else {
dialTo = leverURL.Environment
}
}
conn, err := client.conns.Dial(dialTo)
if err != nil {
return err
}
rpc := &core.RPC{}
if len(args) == 1 {
byteArgs, ok := args[0].([]byte)
if ok {
// Byte args case.
rpc.ArgsOneof = &core.RPC_ByteArgs{
ByteArgs: byteArgs,
}
}
}
if rpc.ArgsOneof == nil {
// Non-byte args case.
var encdArgs *core.JSONArray
encdArgs, err = encodeArgs(args)
if err != nil {
return err
}
rpc.ArgsOneof = &core.RPC_Args{
Args: encdArgs,
}
}
reply, err := core.SendLeverRPC(
conn, context.Background(), leverURL, rpc)
if err != nil {
return err
}
if reply.GetResultOneof() == nil {
return fmt.Errorf("Received nil result oneof")
}
switch result := reply.GetResultOneof().(type) {
case *core.RPCReply_ByteResult:
byteReplyObj, ok := replyObj.(*[]byte)
if !ok {
return fmt.Errorf(
"replyObj needs to be type *[]byte when RPC returns bytes")
}
*byteReplyObj = result.ByteResult
return nil
case *core.RPCReply_ByteError:
return &RemoteByteError{Err: result.ByteError}
case *core.RPCReply_Result:
return decodeArgAsValue(result.Result, replyObj)
case *core.RPCReply_Error:
return &RemoteError{Err: buildDataForJSON(result.Error)}
default:
return fmt.Errorf("Invalid type")
}
}
func (client *Client) invokeChanInternal(
leverURL *core.LeverURL, args ...interface{}) (stream Stream, err error) {
if !IsChanMethod(leverURL.Method) {
return nil, fmt.Errorf(
"Use Invoke / InvokeURL for non-streaming methods")
}
if leverURL.Environment == "" && OwnEnvironment == "" {
return nil, fmt.Errorf(
"Environment not specified and cannot be deduced")
}
if leverURL.Environment == "" {
leverURL.Environment = OwnEnvironment
}
var dialTo string
if client.ForceHost != "" {
dialTo = client.ForceHost
} else {
if (core.IsInternalEnvironment(leverURL.Environment) ||
leverURL.Environment == OwnEnvironment) &&
internalRPCGateway != "" {
dialTo = internalRPCGateway
} else {
dialTo = leverURL.Environment
}
}
conn, err := client.conns.Dial(dialTo)
if err != nil {
return nil, err
}
rpc := &core.RPC{}
if len(args) == 1 {
byteArgs, ok := args[0].([]byte)
if ok {
// Byte args case.
rpc.ArgsOneof = &core.RPC_ByteArgs{
ByteArgs: byteArgs,
}
}
}
if rpc.ArgsOneof == nil {
// Non-byte args case.
var encdArgs *core.JSONArray
encdArgs, err = encodeArgs(args)
if err != nil {
return nil, err
}
rpc.ArgsOneof = &core.RPC_Args{
Args: encdArgs,
}
}
grpcStream, err := core.SendStreamingLeverRPC(
conn, context.Background(), leverURL)
if err != nil {
return nil, err
}
err = grpcStream.Send(&core.StreamMessage{
MessageOneof: &core.StreamMessage_Rpc{
Rpc: rpc,
},
})
if err != nil {
grpcStream.CloseSend()
return nil, err
}
// First message received must be empty.
fstMsg, err := grpcStream.Recv()
if err != nil {
return nil, err
}
if fstMsg.GetMessageOneof() != nil {
return nil, fmt.Errorf("First server message needs to be empty")
}
return newClientStream(grpcStream), nil
}
func (proxy *LeverProxy) handleExtOutStream(
srcEnv string, stream *http2stream.HTTP2Stream, isExtStream bool) {
streamLogger := proxy.outLogger
if isExtStream {
streamLogger = proxy.extLogger
}
headers := stream.GetHeaders()
err := expectHeaders(headers, "lever-url")
if err != nil {
streamLogger.WithFields("err", err).Error("")
stream.Write(&http2stream.MsgError{Err: err})
return
}
leverURL, err := core.ParseLeverURL(headers["lever-url"][0])
if err != nil {
proxy.inLogger.WithFields(
"err", err, "leverURL", headers["lever-url"][0]).Error(
"Unable to parse Lever URL")
}
if !core.IsInternalEnvironment(leverURL.Environment) {
err = fmt.Errorf("Cannot route to dest env")
streamLogger.WithFields(
"err", err,
"leverEnv", leverURL.Environment,
).Error("")
stream.Write(&http2stream.MsgError{Err: err})
return
}
streamID := leverutil.RandomID()
streamLogger.WithFields(
"leverURL", leverURL.String(),
"srcEnv", srcEnv,
"streamID", streamID,
).Debug("Receiving stream")
streamLogger = streamLogger.WithFields("streamID", streamID)
isFromExt := !core.IsInternalEnvironment(srcEnv)
hostInfo, err := proxy.finder.GetHost(
leverURL.Environment, leverURL.Service, leverURL.Resource, isFromExt)
if err != nil {
streamLogger.WithFields(
"err", err,
"leverURL", leverURL.String(),
).Error("Error trying to find / allocate host for request")
stream.Write(&http2stream.MsgError{Err: err})
return
}
streamLogger.WithFields(
"leverInstanceID", hostInfo.InstanceID,
"containerID", hostInfo.ContainerID,
"servingID", hostInfo.ServingID,
"hostProxyAddr", hostInfo.HostAddr,
"isNewInstance", hostInfo.IsNewInstance,
).Debug("Routing to host proxy")
destStream, err := proxy.client.NewStream(hostInfo.HostAddr)
if err != nil {
streamLogger.WithFields(
"err", err,
"hostProxyAddr", hostInfo.HostAddr,
).Error("Error trying to create client stream to dest")
stream.Write(&http2stream.MsgError{Err: err})
return
}
addHeaders := make(map[string][]string)
addHeaders["x-lever-src-env"] = []string{srcEnv}
addHeaders["x-lever-dest-instance-id"] = []string{hostInfo.InstanceID}
addHeaders["x-lever-dest-container-id"] = []string{hostInfo.ContainerID}
addHeaders["x-lever-serving-id"] = []string{hostInfo.ServingID}
addHeaders["x-lever-code-version"] = []string{
strconv.Itoa(int(hostInfo.CodeVersion))}
addHeaders["x-lever-inst-resource-id"] = []string{
hostInfo.LevInstResourceID}
addHeaders["x-lever-inst-session-id"] = []string{hostInfo.LevInstSessionID}
addHeaders["x-lever-res-resource-id"] = []string{hostInfo.LevResResourceID}
addHeaders["x-lever-res-session-id"] = []string{hostInfo.LevResSessionID}
firstHeaders := true
stream.ProxyTo(
destStream,
func(msg http2stream.MsgItem) []http2stream.MsgItem {
proxy.client.KeepAlive(hostInfo.HostAddr)
return proxy.filterTo(&firstHeaders, addHeaders, msg)
},
func(msg http2stream.MsgItem) []http2stream.MsgItem {
proxy.client.KeepAlive(hostInfo.HostAddr)
return noFilter(msg)
})
}