// InvokeChanURL invokes a remote streaming method referenced by a Lever URL.
// The URL can be either absolute (remote environment) or relative
// (same environment).
// Example:
//
// /<service>[/<resource>]/method (relative URL)
//
// lever://<env>/<service>[/<resource>]/method (absolute URL)
func (client *Client) InvokeChanURL(
leverURLStr string, args ...interface{}) (stream Stream, err error) {
leverURL, err := core.ParseLeverURL(leverURLStr)
if err != nil {
return nil, err
}
return client.invokeChanInternal(leverURL, args...)
}
// InvokeURL invokes a remote method referenced by a Lever URL. The URL can
// be either absolute (remote environment) or relative (same environment).
// Example:
//
// /<service>[/<resource>]/method (relative URL)
//
// lever://<env>/<service>[/<resource>]/method (absolute URL)
func (client *Client) InvokeURL(
replyObj interface{}, leverURLStr string, args ...interface{}) (err error) {
leverURL, err := core.ParseLeverURL(leverURLStr)
if err != nil {
return err
}
return client.invokeInternal(replyObj, leverURL, args...)
}
func extractLeverURL(ctx context.Context) (*core.LeverURL, error) {
md, ok := metadata.FromContext(ctx)
if !ok {
return nil, fmt.Errorf("Metadata not found")
}
leverURLStr, ok := md["lever-url"]
if !ok || len(leverURLStr) == 0 {
return nil, fmt.Errorf("Lever URL header not found")
}
return core.ParseLeverURL(leverURLStr[0])
}
// ServeHTTP serves individual HTTP requests.
func (server *Server) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
origin := req.Header.Get("Origin")
if origin != "" {
// TODO: Perhaps a lever.json config param could restrict origin
// to prevent CSRF.
resp.Header().Set("Access-Control-Allow-Origin", origin)
resp.Header().Set(
"Access-Control-Allow-Headers",
"Accept, Content-Type, Content-Length, Accept-Encoding, "+
"X-CSRF-Token, Authorization")
}
if req.Method == "OPTIONS" {
resp.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS")
resp.WriteHeader(http.StatusOK)
return
}
if req.Method != "POST" {
resp.WriteHeader(http.StatusMethodNotAllowed)
return
}
leverURLStr := fmt.Sprintf("lever://%s%s", req.Host, req.URL.Path)
leverURL, err := core.ParseLeverURL(leverURLStr)
if err != nil {
logger.WithFields("err", err).Debug("Error parsing Lever URL")
resp.WriteHeader(http.StatusBadRequest)
return
}
queryValues, err := url.ParseQuery(req.URL.RawQuery)
if err != nil {
logger.WithFields("err", err).Debug("Error parsing URL query")
resp.WriteHeader(http.StatusBadRequest)
return
}
forceEnv := queryValues.Get("forceenv")
if forceEnv != "" {
leverURL.Environment = forceEnv
}
leverURLStr = leverURL.String()
reader := bufferedReaderPool.Get().(*bufio.Reader)
reader.Reset(req.Body)
defer bufferedReaderPool.Put(reader)
defer req.Body.Close()
if leverapi.IsChanMethod(leverURL.Method) {
// TODO: Byte args not supported. Any way to support that?
// How to delimit args from rest?
done := false
var line []byte
line, err = reader.ReadBytes('\n')
if err != nil {
if err == io.EOF {
done = true
} else {
logger.WithFields("err", err).Error("Read error")
return
}
}
if len(line) == 0 {
resp.WriteHeader(http.StatusBadRequest)
return
}
var args []interface{}
err = json.Unmarshal(line, &args)
if err != nil {
resp.WriteHeader(http.StatusBadRequest)
logger.WithFields("err", err).Debug("Malformed JSON")
return
}
var stream leverapi.Stream
stream, err = server.leverClient.InvokeChanURL(leverURLStr, args...)
if err != nil {
resp.WriteHeader(http.StatusInternalServerError)
logger.WithFields("err", err).Error("InvokeChanURL error")
return
}
errCh := make(chan bool)
workerDoneCh := make(chan struct{})
go replyStreamWorker(stream, resp, errCh, workerDoneCh)
if req.Header.Get("Content-Type") == "application/json" {
for !done {
line, err = reader.ReadBytes('\n')
if err != nil {
if err == io.EOF {
done = true
} else {
logger.WithFields("err", err).Error("Read error")
errCh <- true
<-workerDoneCh
return
}
}
if len(line) > 0 {
var msg interface{}
err = json.Unmarshal(line, &msg)
if err != nil {
logger.WithFields("err", err).Debug("Malformed JSON")
errCh <- true
<-workerDoneCh
return
}
stream.Send(msg)
}
}
} else {
for !done {
buffer := bufferPool.Get().([]byte)
defer bufferPool.Put(buffer)
var size int
size, err = reader.Read(buffer)
if err != nil {
if err == io.EOF {
done = true
}
logger.WithFields("err", err).Error("Read error")
errCh <- true
<-workerDoneCh
return
}
msg := buffer[:size]
stream.Send(msg)
}
}
err = stream.Close()
if err != nil {
logger.WithFields("err", err).Debug(
"Stream close error")
errCh <- true
<-workerDoneCh
return
}
errCh <- false
<-workerDoneCh
} else {
buffer := bufferPool.Get().([]byte)
defer bufferPool.Put(buffer)
var size int
size, err = io.ReadFull(reader, buffer)
if err != nil {
if err != io.EOF && err != io.ErrUnexpectedEOF {
logger.WithFields("err", err).Error("Read error")
return
}
}
if size == maxNonChanRequestSize &&
err != io.EOF &&
err != io.ErrUnexpectedEOF {
resp.WriteHeader(http.StatusBadRequest)
_, err = resp.Write([]byte("\"Exceeded maximum request size\""))
if err != nil {
logger.WithFields("err", err).Debug("Write error")
}
return
}
var args []interface{}
contentType := req.Header.Get("Content-Type")
contentTypeSplit := strings.Split(contentType, ";")
switch contentTypeSplit[0] {
case "application/json":
err = json.Unmarshal(buffer[:size], &args)
if err != nil {
resp.WriteHeader(http.StatusBadRequest)
logger.WithFields("err", err).Debug("JSON unmarshal error")
return
}
case "application/x-www-form-urlencoded":
// TODO
resp.WriteHeader(http.StatusBadRequest)
logger.WithFields("contentType", contentType).Error(
"Content type not yet supported")
return
default:
args = make([]interface{}, 1)
args[0] = buffer[:size]
}
var reply interface{}
err = server.leverClient.InvokeURL(&reply, leverURLStr, args...)
if err != nil {
resp.WriteHeader(http.StatusInternalServerError)
remoteErr, ok := err.(*leverapi.RemoteError)
if ok {
reply = remoteErr.Err
} else {
remoteByteErr, ok := err.(*leverapi.RemoteByteError)
if ok {
reply = remoteByteErr.Err
} else {
logger.WithFields("err", err).Error("Internal Lever error")
return
}
}
} else {
resp.WriteHeader(http.StatusOK)
}
err = writeReply(resp, reply)
if err != nil {
logger.WithFields("err", err).Error("Writing reply failed")
return
}
}
}
func (proxy *LeverProxy) handleInStream(stream *http2stream.HTTP2Stream) {
headers := stream.GetHeaders()
err := expectHeaders(
headers,
"lever-url",
"x-lever-src-env",
"x-lever-dest-instance-id",
"x-lever-dest-container-id",
"x-lever-serving-id",
"x-lever-code-version",
"x-lever-inst-resource-id",
"x-lever-inst-session-id",
"x-lever-res-resource-id",
"x-lever-res-session-id",
)
if err != nil {
proxy.inLogger.WithFields("err", err).Error("")
stream.Write(&http2stream.MsgError{Err: err})
return
}
leverURL, err := core.ParseLeverURL(headers["lever-url"][0])
if err != nil {
proxy.inLogger.WithFields(
"err", err, "leverURL", headers["lever-url"][0]).Error(
"Unable to parse Lever URL")
}
if !core.IsInternalEnvironment(leverURL.Environment) {
err = fmt.Errorf("Cannot route to dest env")
proxy.inLogger.WithFields(
"err", err,
"leverEnv", leverURL.Environment,
).Error("")
stream.Write(&http2stream.MsgError{Err: err})
return
}
srcEnv := headers["x-lever-src-env"][0]
instanceID := headers["x-lever-dest-instance-id"][0]
containerID := headers["x-lever-dest-container-id"][0]
servingID := headers["x-lever-serving-id"][0]
codeVersionInt, err := strconv.Atoi(headers["x-lever-code-version"][0])
if err != nil {
proxy.inLogger.WithFields("err", err).Error("Cannot parse code version")
stream.Write(&http2stream.MsgError{Err: err})
return
}
codeVersion := int64(codeVersionInt)
levInstResourceID := headers["x-lever-inst-resource-id"][0]
levInstSessionID := headers["x-lever-inst-session-id"][0]
levResResourceID := headers["x-lever-res-resource-id"][0]
levResSessionID := headers["x-lever-res-session-id"][0]
if instanceID == "" {
instanceID, err = proxy.manager.RandomInstaceID(servingID)
if err != nil {
proxy.inLogger.WithFields(
"err", err,
"leverEnv", leverURL.Environment,
).Error("Could not find an instanceID for provided servingID")
stream.Write(&http2stream.MsgError{Err: err})
return
}
}
streamID := leverutil.RandomID()
proxy.inLogger.WithFields(
"leverURL", leverURL.String(),
"srcEnv", srcEnv,
"leverInstanceID", instanceID,
"containerID", containerID,
"servingID", servingID,
"levInstSessionID", levInstSessionID,
"levInstResourceID", levInstResourceID,
"streamID", streamID,
).Debug("Receiving stream")
streamLogger := proxy.inLogger.WithFields("streamID", streamID)
if !core.IsInternalEnvironment(leverURL.Environment) {
err = fmt.Errorf("Environment not routable internally")
streamLogger.WithFields(
"err", err,
"leverEnv", leverURL.Environment,
).Error("")
stream.Write(&http2stream.MsgError{Err: err})
return
}
_, ownIP, instanceAddr, keepAliveFun, err :=
proxy.manager.EnsureInfrastructureInitialized(&hostman.InstanceInfo{
Environment: leverURL.Environment,
Service: leverURL.Service,
InstanceID: instanceID,
ContainerID: containerID,
ServingID: servingID,
LevInstResourceID: levInstResourceID,
LevInstSessionID: levInstSessionID,
})
if err != nil {
streamLogger.WithFields("err", err).Error(
"Error initializing instance")
stream.Write(&http2stream.MsgError{Err: err})
return
}
err = proxy.serveOut(leverURL.Environment, ownIP)
if err != nil {
streamLogger.WithFields("err", err).Error(
"Error listening on env network")
stream.Write(&http2stream.MsgError{Err: err})
return
}
destStreamI, err := leverutil.ExpBackoff(
func() (clientStream interface{}, err error, finalErr error) {
clientStream, err = proxy.client.NewStream(instanceAddr)
if err != nil {
if strings.Contains(err.Error(), "connection refused") &&
proxy.manager.IsInstanceAlive(servingID, instanceID) {
// Retry.
return nil, err, nil
}
if err == leverutil.ErrNotYetConstructed ||
err == leverutil.ErrWasDestructed {
// Retry.
return nil, err, nil
}
return nil, nil, err
}
return clientStream, nil, nil
}, 10*time.Millisecond, 15*time.Second)
if err != nil {
streamLogger.WithFields(
"err", err,
"instanceAddr", instanceAddr,
).Error("Error trying to create client stream to dest")
stream.Write(&http2stream.MsgError{Err: err})
return
}
destStream := destStreamI.(*http2stream.HTTP2Stream)
addHeaders := make(map[string][]string)
if srcEnv != "" {
addHeaders["x-lever-src-env"] = []string{srcEnv}
}
addHeaders["x-lever-internal-rpc-gateway"] = []string{
ownIP + ":" + EnvOutListenPortFlag.Get()}
startTime := time.Now()
firstHeaders := true
stream.ProxyTo(
destStream,
func(msg http2stream.MsgItem) []http2stream.MsgItem {
proxy.client.KeepAlive(instanceAddr)
keepAliveFun(
leverURL.Resource, levResResourceID, levResSessionID)
return proxy.filterTo(&firstHeaders, addHeaders, msg)
},
func(msg http2stream.MsgItem) []http2stream.MsgItem {
proxy.client.KeepAlive(instanceAddr)
keepAliveFun(
leverURL.Resource, levResResourceID, levResSessionID)
return noFilter(msg)
})
// Wait for RPC to finish.
<-destStream.Closed()
rpcNanos := uint64(time.Now().Sub(startTime).Nanoseconds())
streamLogger.WithFields("rpcNanos", rpcNanos).Debug("RPC nanos")
// Send RPC stats to fleettracker.
// TODO: For services with high load, we should not send info on every
// RPC. They should be batched and sent say... every ~50ms
// (well below tracker tick interval).
err = fleettracker.OnRPC(proxy.grpcPool, &fleettracker.RPCEvent{
Environment: leverURL.Environment,
Service: leverURL.Service,
ServingID: servingID,
CodeVersion: codeVersion,
IsAdmin: core.IsAdmin(leverURL),
RpcNanos: rpcNanos,
})
if err != nil {
streamLogger.WithFields("err", err).Error(
"Failed to send RPC stats to fleettracker")
}
}
func actionStream(ctx *cli.Context) {
leverURL, err := core.ParseLeverURL(ctx.Args().Get(0))
if err != nil {
logger.WithFields("err", err).Fatal("Invalid lever URL")
}
if leverURL.Environment == "" {
leverURL.Environment = flagEnv
}
client, err := leverapi.NewClient()
if err != nil {
logger.WithFields("err", err).Fatal("Error creating client")
}
if strings.HasSuffix(leverURL.Environment, ".lever") {
client.ForceHost = detectLeverOSIPPort()
}
if flagHost != "" {
client.ForceHost = flagHost
}
var args []interface{}
for index := 1; index < ctx.NArg(); index++ {
rawArg := ctx.Args().Get(index)
var arg interface{}
err = json.Unmarshal([]byte(rawArg), &arg)
if err != nil {
logger.WithFields("err", err, "jsonArg", rawArg).Fatal(
"Error parsing JSON arg")
}
args = append(args, arg)
}
stream, err := client.InvokeChanURL(leverURL.String(), args...)
if err != nil {
logger.WithFields("err", err).Fatal("Invokation error")
}
receiveDone := make(chan struct{})
go func() {
stdout := bufio.NewWriter(os.Stdout)
for {
var msg interface{}
err = stream.Receive(&msg)
if err == io.EOF {
break
}
if err != nil {
logger.WithFields("err", err).Fatal(
"Error receiving from server")
}
msgBytes, isBytes := msg.([]byte)
if isBytes {
_, err = stdout.Write(msgBytes)
if err != nil {
logger.WithFields("err", err).Fatal(
"Error writing to stdout")
}
} else {
var jsonMsg []byte
if flagPrettyPrint {
jsonMsg, err = json.MarshalIndent(msg, "", " ")
} else {
jsonMsg, err = json.Marshal(msg)
}
if err != nil {
logger.WithFields("err", err).Fatal(
"Error turning receiving msg into JSON")
}
stdout.Write(jsonMsg)
stdout.WriteString("\n")
}
}
err = stdout.Flush()
if err != nil {
logger.WithFields("err", err).Fatal("Error flushing stdout")
}
close(receiveDone)
}()
stdin := bufio.NewReader(os.Stdin)
if flagBytes {
chunk := make([]byte, 32*1024)
for {
var size int
size, err = stdin.Read(chunk)
if err == io.EOF {
break
}
if err != nil {
logger.WithFields("err", err).Fatal("Error reading from stdin")
}
err = stream.Send(chunk[:size])
if err != nil {
logger.WithFields("err", err).Fatal("Error sending chunk")
}
}
} else {
for {
var line string
line, err = stdin.ReadString('\n')
if err == io.EOF && line == "" {
break
}
if err != nil && err != io.EOF {
logger.WithFields("err", err).Fatal("Error reading from stdin")
}
var jsonMsg interface{}
err = json.Unmarshal([]byte(line), &jsonMsg)
if err != nil {
logger.WithFields("err", err).Fatal(
"Error parsing JSON from stdin")
}
err = stream.Send(jsonMsg)
if err != nil {
logger.WithFields("err", err).Fatal("Error sending JSON")
}
}
}
err = stream.Close()
if err != nil {
logger.WithFields("err", err).Fatal("Error trying to close stream")
}
<-receiveDone
}
func actionInvoke(ctx *cli.Context) {
leverURL, err := core.ParseLeverURL(ctx.Args().Get(0))
if err != nil {
logger.WithFields("err", err).Fatal("Invalid lever URL")
}
if leverURL.Environment == "" {
leverURL.Environment = flagEnv
}
client, err := leverapi.NewClient()
if err != nil {
logger.WithFields("err", err).Fatal("Error creating client")
}
if strings.HasSuffix(leverURL.Environment, ".lever") {
client.ForceHost = detectLeverOSIPPort()
}
if flagHost != "" {
client.ForceHost = flagHost
}
if ctx.NArg() > 1 && ctx.Args().Get(1) == "--" {
// Byte args case.
// TODO: Read bytes arg from stdin.
logger.Fatal("Bytes from stdin not yet implemented")
return
}
// JSON args case.
var args []interface{}
for index := 1; index < ctx.NArg(); index++ {
rawArg := ctx.Args().Get(index)
var arg interface{}
err = json.Unmarshal([]byte(rawArg), &arg)
if err != nil {
logger.WithFields("err", err, "jsonArg", rawArg).Fatal(
"Error parsing JSON arg")
}
args = append(args, arg)
}
var reply interface{}
err = client.InvokeURL(&reply, leverURL.String(), args...)
if err != nil {
logger.WithFields("err", err).Fatal("Invokation error")
}
bytes, isByteReply := reply.([]byte)
if !isByteReply {
if flagPrettyPrint {
bytes, err = json.MarshalIndent(reply, "", " ")
} else {
bytes, err = json.Marshal(reply)
}
if err != nil {
logger.WithFields("err", err).Fatal(
"Error turning reply into JSON form")
}
}
stdout := bufio.NewWriter(os.Stdout)
stdout.Write(bytes)
if !isByteReply && flagPrettyPrint {
stdout.WriteString("\n")
}
stdout.Flush()
}
func (proxy *LeverProxy) handleExtOutStream(
srcEnv string, stream *http2stream.HTTP2Stream, isExtStream bool) {
streamLogger := proxy.outLogger
if isExtStream {
streamLogger = proxy.extLogger
}
headers := stream.GetHeaders()
err := expectHeaders(headers, "lever-url")
if err != nil {
streamLogger.WithFields("err", err).Error("")
stream.Write(&http2stream.MsgError{Err: err})
return
}
leverURL, err := core.ParseLeverURL(headers["lever-url"][0])
if err != nil {
proxy.inLogger.WithFields(
"err", err, "leverURL", headers["lever-url"][0]).Error(
"Unable to parse Lever URL")
}
if !core.IsInternalEnvironment(leverURL.Environment) {
err = fmt.Errorf("Cannot route to dest env")
streamLogger.WithFields(
"err", err,
"leverEnv", leverURL.Environment,
).Error("")
stream.Write(&http2stream.MsgError{Err: err})
return
}
streamID := leverutil.RandomID()
streamLogger.WithFields(
"leverURL", leverURL.String(),
"srcEnv", srcEnv,
"streamID", streamID,
).Debug("Receiving stream")
streamLogger = streamLogger.WithFields("streamID", streamID)
isFromExt := !core.IsInternalEnvironment(srcEnv)
hostInfo, err := proxy.finder.GetHost(
leverURL.Environment, leverURL.Service, leverURL.Resource, isFromExt)
if err != nil {
streamLogger.WithFields(
"err", err,
"leverURL", leverURL.String(),
).Error("Error trying to find / allocate host for request")
stream.Write(&http2stream.MsgError{Err: err})
return
}
streamLogger.WithFields(
"leverInstanceID", hostInfo.InstanceID,
"containerID", hostInfo.ContainerID,
"servingID", hostInfo.ServingID,
"hostProxyAddr", hostInfo.HostAddr,
"isNewInstance", hostInfo.IsNewInstance,
).Debug("Routing to host proxy")
destStream, err := proxy.client.NewStream(hostInfo.HostAddr)
if err != nil {
streamLogger.WithFields(
"err", err,
"hostProxyAddr", hostInfo.HostAddr,
).Error("Error trying to create client stream to dest")
stream.Write(&http2stream.MsgError{Err: err})
return
}
addHeaders := make(map[string][]string)
addHeaders["x-lever-src-env"] = []string{srcEnv}
addHeaders["x-lever-dest-instance-id"] = []string{hostInfo.InstanceID}
addHeaders["x-lever-dest-container-id"] = []string{hostInfo.ContainerID}
addHeaders["x-lever-serving-id"] = []string{hostInfo.ServingID}
addHeaders["x-lever-code-version"] = []string{
strconv.Itoa(int(hostInfo.CodeVersion))}
addHeaders["x-lever-inst-resource-id"] = []string{
hostInfo.LevInstResourceID}
addHeaders["x-lever-inst-session-id"] = []string{hostInfo.LevInstSessionID}
addHeaders["x-lever-res-resource-id"] = []string{hostInfo.LevResResourceID}
addHeaders["x-lever-res-session-id"] = []string{hostInfo.LevResSessionID}
firstHeaders := true
stream.ProxyTo(
destStream,
func(msg http2stream.MsgItem) []http2stream.MsgItem {
proxy.client.KeepAlive(hostInfo.HostAddr)
return proxy.filterTo(&firstHeaders, addHeaders, msg)
},
func(msg http2stream.MsgItem) []http2stream.MsgItem {
proxy.client.KeepAlive(hostInfo.HostAddr)
return noFilter(msg)
})
}