func (suite *sessionRecoverySuite) TestExpiredCacheRecovery() {
scope := New().(*realScope)
scope.userCache = newTestCache()
mock := multiclient.NewMock()
stub := &multiclient.Stub{
Service: loginService,
Endpoint: readSessionEndpoint,
Response: &sessreadproto.Response{
SessId: proto.String(testSessId),
Token: proto.String(testToken),
},
}
mock.Stub(stub)
multiclient.SetCaller(mock.Caller())
u, err := FromSessionToken(testSessId, testToken)
suite.Assertions.Equal(testSessId, u.SessId)
suite.Assertions.NoError(err)
suite.Assertions.NotNil(u)
u.RenewTs = time.Now().Add(-2 * time.Minute)
u.ExpiryTs = u.RenewTs.Add(1 * time.Minute)
suite.Assertions.NoError(scope.userCache.Store(u))
suite.Assertions.Equal(0, stub.CountCalls())
// As this session has expired, the login service should be called to recover (and renew) it
suite.Assertions.False(scope.IsAuth())
suite.Assertions.NoError(scope.RecoverSession(testSessId))
suite.Assertions.True(scope.IsAuth())
u = scope.AuthUser()
suite.Assertions.NotNil(u)
suite.Assertions.Equal(1, stub.CountCalls())
}
// ToProtobuf takes a Error and returns a protobuf error
func ToProtobuf(err Error) *pe.PlatformError {
return &pe.PlatformError{
Type: pe.PlatformError_ErrorType(pe.PlatformError_ErrorType_value[err.Type()]).Enum(),
Code: proto.String(err.Code()),
Description: proto.String(err.Description()),
Context: err.Context(),
HttpCode: proto.Uint32(err.HttpCode()),
}
}
func healthCheckSampleToProto(hc *HealthCheck, sample *Sample) *hcproto.HealthCheck {
return &hcproto.HealthCheck{
Timestamp: proto.Int64(sample.At.Unix()),
HealthCheckId: proto.String(hc.Id),
ServiceName: proto.String(hc.ServiceName),
ServiceVersion: proto.Uint64(hc.ServiceVersion),
Hostname: proto.String(hc.Hostname),
InstanceId: proto.String(hc.InstanceId),
IsHealthy: proto.Bool(sample.IsHealthy),
ErrorDescription: proto.String(sample.ErrorDescription),
Measurements: mapToProto(sample.Measurements),
Priority: hcproto.HealthCheck_Priority(hc.Priority).Enum(),
}
}
func mapToProto(m map[string]string) []*hcproto.HealthCheck_KeyValue {
if m == nil {
return []*hcproto.HealthCheck_KeyValue{}
}
ret := make([]*hcproto.HealthCheck_KeyValue, len(m))
i := 0
for k, v := range m {
ret[i] = &hcproto.HealthCheck_KeyValue{
Key: proto.String(k),
Value: proto.String(v),
}
i++
}
return ret
}
// SignOut destroys the current session so that it cannot be used again
func (s *realScope) SignOut(user *User) error {
cl := multiclient.New().DefaultScopeFrom(s.getRpcScope())
cl.AddScopedReq(&multiclient.ScopedReq{
Uid: "deletesess",
Service: loginService,
Endpoint: deleteSessionEndpoint,
Req: &sessdelproto.Request{
SessId: proto.String(user.SessId),
},
Rsp: &sessdelproto.Response{},
})
if cl.Execute().AnyErrors() {
return cl.Succeeded("deletesess")
}
if err := s.userCache.Purge(user.SessId); err != nil {
log.Errorf("[Auth] Error purging session cache: %v", err)
}
s.Lock()
defer s.Unlock()
s.authUser = nil
s.triedAuth = false
return nil
}
// jsonschemaHandler returns all registered endpoints in json schema format as per ITF draft4
// http://json-schema.org/latest/json-schema-core.html
func jsonschemaHandler(req *Request) (proto.Message, errors.Error) {
// Get all endpoints
request := req.Data().(*jsonschemaproto.Request)
endpoint := request.GetEndpoint()
endpoints := reg.iterate()
schemas := make([]*jsonschema.JsonSchema, 0)
for _, ep := range endpoints {
if endpoint != "" && endpoint != ep.GetName() {
continue
}
schema, err := marshalEndpoint(ep)
if err == nil && schema != nil {
schemas = append(schemas, schema)
}
}
rsp, err := json.Marshal(schemas)
if err != nil {
return nil, errors.InternalServerError("com.hailocab.kernel.marshal.error", fmt.Sprintf("Unable to unmarshal response data: %v", err.Error()))
}
return &jsonschemaproto.Response{
Jsonschema: proto.String(string(rsp)),
}, nil
}
func (s *realScope) doAuth(mech, device string, creds map[string]string) (*User, error) {
reqProto := &authproto.Request{
Mech: proto.String(mech),
DeviceType: proto.String(device),
Meta: make([]*loginproto.KeyValue, 0),
}
for k, v := range creds {
switch k {
case "username":
reqProto.Username = proto.String(v)
case "password":
reqProto.Password = proto.String(v)
case "newPassword":
reqProto.NewPassword = proto.String(v)
case "application":
reqProto.Application = proto.String(v)
default:
// Add additional fields to Meta, such as DeviceId, osVersion, appVersion
reqProto.Meta = append(reqProto.Meta, &loginproto.KeyValue{
Key: proto.String(k),
Value: proto.String(v),
})
}
}
cl := multiclient.New().DefaultScopeFrom(s.getRpcScope())
rsp := &authproto.Response{}
cl.AddScopedReq(&multiclient.ScopedReq{
Uid: "auth",
Service: loginService,
Endpoint: authEndpoint,
Req: reqProto,
Rsp: rsp,
Options: client.Options{"retries": 0},
})
if cl.Execute().AnyErrors() {
// specfically map out bad credentials error
err := cl.Succeeded("auth")
if err.Code() == badCredentialsErrCode {
return nil, BadCredentialsError
}
return nil, err
}
// recover this user
u, err := FromSessionToken(rsp.GetSessId(), rsp.GetToken())
if err != nil {
return nil, err
}
if err := s.userCache.Store(u); err != nil {
log.Errorf("[Auth] Error caching session: %v", err)
}
return u, nil
}
func (s *stats) endpoint(ep Endpoint, t endpointTimers) *pstats.EndpointStats {
return &pstats.EndpointStats{
EndpointName: proto.String(ep.GetName()),
Sla: endpointSLA(ep),
Success: endpointStat(t.success),
Error: endpointStat(t.errors),
}
}
// publishFailure publishes a failure/panic event to be monitored.
func publishFailure(r interface{}) {
var p string
switch r.(type) {
case string:
p = r.(string)
case error:
p = fmt.Sprintf("%v", r.(error))
default:
p = "Unknown panic"
}
b := make([]byte, 1024)
runtime.Stack(b, true)
if err := client.Pub("com.hailocab.monitor.failure", &fproto.Failure{
ServiceName: proto.String(Name),
ServiceVersion: proto.Uint64(Version),
AzName: proto.String(az),
Hostname: proto.String(hostname),
InstanceId: proto.String(InstanceID),
Timestamp: proto.Int64(time.Now().Unix()),
Uptime: proto.Int64(int64(time.Since(serviceStarted).Seconds())),
Type: proto.String("PANIC"),
Reason: proto.String(p),
Stack: proto.String(string(b)),
}); err != nil {
log.Errorf("[Server] Failed to publish failure event: %v", err)
}
}
func TestResponder(t *testing.T) {
stub := &Stub{
Service: mockFooService,
Endpoint: mockHealthEndpoint,
Responder: func(invocation int, req *client.Request) (proto.Message, errors.Error) {
if invocation == 1 {
return &hcproto.Response{
Healthchecks: []*hcproto.HealthCheck{
&hcproto.HealthCheck{
Timestamp: proto.Int64(1403629015),
ServiceName: proto.String("foo"),
ServiceVersion: proto.Uint64(1403629015),
Hostname: proto.String("localhost"),
InstanceId: proto.String("foobar"),
HealthCheckId: proto.String("boom"),
IsHealthy: proto.Bool(true),
},
},
}, nil
}
return nil, errors.InternalServerError("only.one.allowed", "First call only works")
},
}
mock := NewMock().Stub(stub)
caller := mock.Caller()
req, _ := client.NewRequest(mockFooService, mockHealthEndpoint, &hcproto.Request{})
rsp := &hcproto.Response{}
e := caller(req, rsp)
assert.Nil(t, e,
"Expecting our mocked call to be intercepted and stubbed response returned, got err: %v", e)
assert.Len(t, rsp.GetHealthchecks(), 1,
"Response does not contain our mocked content: no healthchecks")
// now repeat, and we SHOULD get an error
e = caller(req, rsp)
assert.NotNil(t, e,
"Expecting our mocked call to be intercepted and error response returned on 2nd call")
assert.Equal(t, e.Code(), "only.one.allowed",
"Expecting code 'only.one.allowed', got '%s'", e.Code())
}
// doRecoverSession is the meat and veg for RecoverSession
func (s *realScope) doRecoverSession(sessId string) (*User, error) {
// Check cache; ignore errors (will have impact on service performance, but not functionality)
queryLogin := false
u, hit, err := s.userCache.Fetch(sessId)
if err != nil {
log.Warnf("[Auth] Error fetching session from cache (will call login service): %v", err)
queryLogin = true
} else if u != nil && u.ExpiryTs.Before(time.Now()) && u.CanAutoRenew() { // Cached token has expired
log.Infof("[Auth] Cache-recovered token has expired (%s); will call login service", u.ExpiryTs.String())
queryLogin = true
} else {
queryLogin = u == nil && !hit
}
if queryLogin {
cl := multiclient.New().DefaultScopeFrom(s.getRpcScope())
rsp := &sessreadproto.Response{}
cl.AddScopedReq(&multiclient.ScopedReq{
Uid: "readsess",
Service: loginService,
Endpoint: readSessionEndpoint,
Req: &sessreadproto.Request{
SessId: proto.String(sessId),
},
Rsp: rsp,
})
if cl.Execute().AnyErrorsIgnoring([]string{errors.ErrorNotFound}, nil) {
err := cl.Succeeded("readsess")
log.Errorf("[Auth] Auth scope recovery error [%s: %s] %v", err.Type(), err.Code(), err.Description())
return nil, err
}
// found a session?
if rsp.GetSessId() == "" && rsp.GetToken() == "" {
log.Debugf("[Auth] Session '%s' not found (not valid) when trying to recover from login service", sessId)
// @todo we could cache this (at least for a short time) to prevent repeated hammering of login service
} else {
u, err = FromSessionToken(rsp.GetSessId(), rsp.GetToken())
if err != nil {
log.Errorf("[Auth] Error getting user from session: %v", err)
} else {
log.Tracef("[Auth] Auth scope - recovered user '%s' from session '%s'", u.Id, rsp.GetSessId())
}
}
// ignore errors; just means we have no user
if u != nil {
s.userCache.Store(u)
}
}
return u, nil
}
// TestRecoverSessionHappyFound tests happy case (no service call failures) when we do find a user
func (suite *sessionRecoverySuite) TestRecoverSessionHappyFound() {
scope := New().(*realScope)
scope.userCache = newTestCache()
mock := multiclient.NewMock()
stub := &multiclient.Stub{
Service: loginService,
Endpoint: readSessionEndpoint,
Response: &sessreadproto.Response{
SessId: proto.String(testSessId),
Token: proto.String(testToken),
},
}
mock.Stub(stub)
multiclient.SetCaller(mock.Caller())
err := scope.RecoverSession(testSessId)
suite.Assertions.NoError(err, "Unexpected recovery error")
suite.Assertions.True(scope.IsAuth())
suite.Assertions.True(scope.HasTriedAuth())
suite.Assertions.Equal(1, stub.CountCalls(), "Expecting 1 call to readsession")
req := &sessreadproto.Request{}
err = stub.Request(0).Unmarshal(req)
suite.Assertions.NoError(err)
suite.Assertions.Equal(testSessId, req.GetSessId())
suite.Assertions.False(req.GetNoRenew())
u := scope.authUser
suite.Assertions.NotNil(u)
suite.Assertions.Equal("dave", u.Id)
// Clean out scope
scope.Clean()
suite.Assertions.False(scope.IsAuth(), "Expecting scope to be IsAuth==false after Clean()")
suite.Assertions.False(scope.HasTriedAuth(), "Expecting scope to have HasTriedAuth()==false after Clean()")
suite.Assertions.Nil(scope.AuthUser(), "Expecting AuthUser()==nil after Clean()")
// recover AGAIN -- this time it shoud be cached
suite.Assertions.NoError(scope.RecoverSession(testSessId), "Unexpected recovery error")
suite.Assertions.True(scope.IsAuth())
suite.Assertions.Equal(1, stub.CountCalls(), "Expecting 1 call to readsession (should be cached now)")
}
// reloadSlas loads timeouts from discovery service for all services we know about (have tried to call)
func (t *Timeout) reloadSlas() {
replacement := make(map[string]map[string]time.Duration)
for service := range t.endpoints {
// load from discovery service
log.Debugf("[Client] Loading SLAs from discovery service for %v...", service)
req, err := NewRequest("com.hailocab.kernel.discovery", "endpoints", &eps.Request{
Service: proto.String(service),
})
if err != nil {
log.Warnf("[Client] Failed to create proto request to get endpoints for service: %s", service)
continue
}
rsp := &eps.Response{}
// explicitly define timeout since we're in no rush
if err := t.client.Req(req, rsp, Options{"retries": 0, "timeout": time.Second * 5}); err != nil {
log.Warnf("[Client] Trouble getting endpoint response back from discovery-service for service: %s", service)
continue
}
for _, ep := range rsp.GetEndpoints() {
endpoint := strings.TrimLeft(strings.TrimPrefix(ep.GetFqName(), service), ".")
if _, ok := replacement[service]; !ok {
replacement[service] = make(map[string]time.Duration)
}
replacement[service][endpoint] = msToDuration(ep.GetUpper95())
}
}
// double check we have all the things we started with -- if not, but back the "last known" (probably defaults)
for service, serviceEndpoints := range t.endpoints {
for endpoint, timeout := range serviceEndpoints {
if _, ok := replacement[service]; !ok {
replacement[service] = make(map[string]time.Duration)
}
if _, ok := replacement[service][endpoint]; !ok {
log.Debugf("[Client] Failed to find SLA for %s.%s, falling back to %v", service, endpoint, timeout)
replacement[service][endpoint] = timeout
}
}
}
// SLAs changed? if not, don't bother switching+logging
if hashSlas(replacement) == t.hashEndpoints() {
return
}
t.Lock()
defer t.Unlock()
t.endpoints = replacement
log.Infof("[Client] Loaded new SLAs from discovery service: %v", t.endpoints)
}
// traceIn traces a request inbound to a service to handle
func traceIn(req *Request) {
if req.shouldTrace() {
go trace.Send(&traceproto.Event{
Timestamp: proto.Int64(time.Now().UnixNano()),
TraceId: proto.String(req.TraceID()),
Type: traceproto.Event_IN.Enum(),
MessageId: proto.String(req.MessageID()),
ParentMessageId: proto.String(req.ParentMessageID()),
From: proto.String(req.From()),
To: proto.String(fmt.Sprintf("%v.%v", req.Service(), req.Endpoint())),
Hostname: proto.String(hostname),
Az: proto.String(az),
Payload: proto.String(""), // @todo
HandlerInstanceId: proto.String(InstanceID),
PersistentTrace: proto.Bool(req.TraceShouldPersist()),
})
}
}
// traceReq decides if we want to trigger a trace event (when sending a request) and if so deals with it
func (c *client) traceReq(req *Request) {
if req.shouldTrace() {
trace.Send(&traceproto.Event{
Timestamp: proto.Int64(time.Now().UnixNano()),
TraceId: proto.String(req.TraceID()),
Type: traceproto.Event_REQ.Enum(),
MessageId: proto.String(req.MessageID()),
ParentMessageId: proto.String(req.ParentMessageID()),
From: proto.String(req.From()),
FromEndpoint: proto.String(req.FromEndpoint()),
To: proto.String(fmt.Sprintf("%v.%v", req.Service(), req.Endpoint())),
Hostname: proto.String(c.hostname),
Az: proto.String(c.az),
Payload: proto.String(""), // @todo
PersistentTrace: proto.Bool(req.TraceShouldPersist()),
})
}
}
func Foo(req h2.Request) (proto.Message, h2.Error) {
request := req.Data().(*foo.Request)
log.Debugf("Received bar=%v", request.GetBar())
s := services.Response{}
err := h2.Call("com.hailocab.kernel.discovery", "services", &services.Request{}, &s)
if err != nil {
log.Warnf("Ouch... the discovery services seems to be down ): %v", err)
}
rsp := &foo.Response{
Baz: proto.String(fmt.Sprintf("There are %v services running on your h2 cluster", len(s.GetServices()))),
}
return rsp, nil
}
// load config via login service
func (s *serviceToService) load() error {
svc := s.getService()
if svc == "" {
log.Debug("[Auth] Skipping loading service-to-service auth rules (no service defined)")
return nil
}
log.Tracef("[Auth] Loading service-to-service auth rules for %s", svc)
reqProto := &endpointauth.Request{
Service: proto.String(svc),
}
req, err := client.NewRequest("com.hailocab.service.login", "endpointauth", reqProto)
if err != nil {
return err
}
// scope it
req.SetFrom(svc)
rsp := &endpointauth.Response{}
if err := client.Req(req, rsp); err != nil {
return err
}
newEndpoints := make(map[string]grantedServices)
for _, ep := range rsp.GetEndpoints() {
name := ep.GetEndpoint()
if _, ok := newEndpoints[name]; !ok {
newEndpoints[name] = make(grantedServices)
}
// add in the granted services to this endpoint
for _, gs := range ep.GetGranted() {
newEndpoints[name][gs.GetName()] = role(gs.GetRole())
}
}
// check if changed - to avoid locking/changing/logging if not
if hashEndpoints(newEndpoints) == s.hash() {
return nil
}
// switch in config
s.Lock()
defer s.Unlock()
s.endpoints = newEndpoints
log.Debugf("[Auth] Loaded service-to-service auth rules: %#v", s.endpoints)
return nil
}
func Endpoint(name string, configStruct interface{}) *service.Endpoint {
handler := func(req *server.Request) (proto.Message, errors.Error) {
return &schemaProto.Response{
Schema: proto.String(schema.Of(configStruct).String()),
}, nil
}
return &server.Endpoint{
Name: name,
Mean: 200,
Upper95: 400,
Handler: handler,
Authoriser: service.OpenToTheWorldAuthoriser(),
}
}
// Hello provides a func to process an incoming 'hello' request.
// A handler requires a request parameter and will return either a response or an error
func Hello(req h2.Request) (proto.Message, h2.Error) {
request := req.Data().(*protoHello.Request)
// Error example
if len(request.GetName()) == 0 {
// We specify multi error types in the platform layer repository in the 'errors'
// package. See: https://godoc.org/github.com/hailocab/platform-layer/errors
return nil, errors.BadRequest(HelloEndpoint, "You didn't specify a name in the request")
}
// Return response with the message set to `Hello <name>`
return &protoHello.Response{
Message: proto.String(fmt.Sprintf("Hello %q", request.GetName())),
}, nil
}
func TestMockCallerPopulatesResponse(t *testing.T) {
req, _ := client.NewRequest(mockFooService, mockHealthEndpoint, &hcproto.Request{})
stub := &Stub{
Service: mockFooService,
Endpoint: mockHealthEndpoint,
Response: &hcproto.Response{
Healthchecks: []*hcproto.HealthCheck{
&hcproto.HealthCheck{
Timestamp: proto.Int64(1403629015),
ServiceName: proto.String("foo"),
ServiceVersion: proto.Uint64(1403629015),
Hostname: proto.String("localhost"),
InstanceId: proto.String("foobar"),
HealthCheckId: proto.String("boom"),
IsHealthy: proto.Bool(true),
},
},
},
}
mock := NewMock().Stub(stub)
caller := mock.Caller()
rsp := &hcproto.Response{}
e := caller(req, rsp)
assert.Nil(t, e,
"Expecting our mocked call to be intercepted and stubbed response returned, got err: %v", e)
// ensure stub has what we expect
assert.Len(t, stub.matched, 1,
"Expecting 1 match payload to be stored after execution")
assert.Equal(t, stub.CountCalls(), 1, "CountCalls should return 1 too")
assert.Len(t, rsp.GetHealthchecks(), 1,
"Response does not contain our mocked content: no healthchecks")
}
// get returns a snapshot of platform stats
func (s *stats) get(status string) *pstats.PlatformStats {
rusageStats := s.rusage()
runtimeStats := s.runtime()
endpointStats := s.endpoints()
return &pstats.PlatformStats{
ServiceName: proto.String(ServiceName),
ServiceVersion: proto.Uint64(ServiceVersion),
ServiceType: proto.String(ServiceType),
AzName: proto.String(AzName),
Hostname: proto.String(hostname),
InstanceId: proto.String(InstanceID),
Status: proto.String(status),
Timestamp: proto.Int64(time.Now().Unix()),
Uptime: proto.Int64(int64(time.Since(s.startTime).Seconds())),
Rusage: rusageStats,
Runtime: runtimeStats,
Endpoints: endpointStats,
}
}
// ConfiguredHttpCaller with more explicit configuration options than simple HttpCaller
func ConfiguredHttpCaller(opts Options) Caller {
tp := &httpclient.Transport{
ConnectTimeout: durationOrDefault(opts.ConnectTimeout, 5*time.Second),
RequestTimeout: durationOrDefault(opts.RequestTimeout, 5*time.Second),
ResponseHeaderTimeout: durationOrDefault(opts.ResponseHeaderTimeout, 5*time.Second),
}
if opts.TlsSkipVerify {
tp.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
httpClient := &http.Client{Transport: tp}
return func(req *client.Request, rsp proto.Message) errors.Error {
u, err := url.Parse(opts.BaseUrl)
q := u.Query()
q.Set("session_id", req.SessionID())
q.Set("service", req.Service())
q.Set("endpoint", req.Endpoint())
u.Path = "/rpc"
u.RawQuery = q.Encode()
var httpReq *http.Request
// send JSON req content-type to thin API as form-encoded data
// send proto req content-type directly as bytes, with proto content type
if req.ContentType() == jsonContentType {
values := make(url.Values)
values.Set("service", req.Service())
values.Set("endpoint", req.Endpoint())
values.Set("request", string(req.Payload()))
httpReq, _ = http.NewRequest("POST", u.String(), bytes.NewReader([]byte(values.Encode())))
httpReq.Header.Set("Content-Type", formEncodedContentType)
} else {
httpReq, _ = http.NewRequest("POST", u.String(), bytes.NewReader(req.Payload()))
httpReq.Header.Set("Content-Type", protoContentType)
}
log.Tracef("[Multiclient] HTTP caller - calling '%s' : content-type '%s'", u.String(), req.ContentType())
httpRsp, err := httpClient.Do(httpReq)
if err != nil {
log.Warnf("[Multiclient] HTTP caller error calling %s.%s via %s : %s", req.Service(), req.Endpoint(), u.String(), err)
return errors.InternalServerError("multiclienthttp.postform", fmt.Sprintf("Error calling %s.%s via %s : %s", req.Service(), req.Endpoint(), u.String(), err))
}
defer httpRsp.Body.Close()
rspBody, err := ioutil.ReadAll(httpRsp.Body)
if err != nil {
return errors.BadResponse("multiclienthttp.readresponse", fmt.Sprintf("Error reading response bytes: %v", err))
}
// what status code?
if httpRsp.StatusCode != 200 {
// deal with error
e := &protoerror.PlatformError{}
var err error
if req.ContentType() == jsonContentType {
jsonErr := &errorBody{}
err = json.Unmarshal(rspBody, jsonErr)
e.Code = proto.String(jsonErr.DottedCode)
e.Context = jsonErr.Context
e.Description = proto.String(jsonErr.Payload)
e.HttpCode = proto.Uint32(uint32(httpRsp.StatusCode))
// this conversion is lossy, since the JSON response for errors, as crafted
// by the "thin API", does not currently include the error type, so we have
// to guess from HTTP status code, but there is no distinct code for "BAD_RESPONSE"
switch httpRsp.StatusCode {
case 400:
e.Type = protoerror.PlatformError_BAD_REQUEST.Enum()
case 403:
e.Type = protoerror.PlatformError_FORBIDDEN.Enum()
case 404:
e.Type = protoerror.PlatformError_NOT_FOUND.Enum()
case 500:
e.Type = protoerror.PlatformError_INTERNAL_SERVER_ERROR.Enum()
case 504:
e.Type = protoerror.PlatformError_TIMEOUT.Enum()
}
} else {
err = proto.Unmarshal(rspBody, e)
}
// some issue understanding error rsp
if err != nil {
return errors.BadResponse("multiclienthttp.unmarshalerr", fmt.Sprintf("Error unmarshaling error response '%s': %v", string(rspBody), err))
}
return errors.FromProtobuf(e)
}
// unmarshal response
if req.ContentType() == jsonContentType {
err = json.Unmarshal(rspBody, rsp)
} else {
err = proto.Unmarshal(rspBody, rsp)
}
if err != nil {
return errors.BadResponse("multiclienthttp.unmarshal", fmt.Sprintf("Error unmarshaling response: %v", err))
}
return nil
}
}
// traceAttemptTimeout decides if we want to trigger a trace event for an attempt timeout, and processes it
func (c *client) traceAttemptTimeout(req *Request, attemptNum int, timeout time.Duration) {
if req.shouldTrace() {
desc := fmt.Sprintf("Attempt %v timeout talking to '%s.%s' after '%v' for '%s'", attemptNum, req.Service(), req.Endpoint(), timeout, req.MessageID())
trace.Send(&traceproto.Event{
Timestamp: proto.Int64(time.Now().UnixNano()),
TraceId: proto.String(req.TraceID()),
Type: traceproto.Event_ATTEMPT_TIMEOUT.Enum(),
MessageId: proto.String(req.MessageID()),
From: proto.String(req.From()),
FromEndpoint: proto.String(req.FromEndpoint()),
To: proto.String(fmt.Sprintf("%v.%v", req.Service(), req.Endpoint())),
ParentMessageId: proto.String(req.ParentMessageID()),
Hostname: proto.String(c.hostname),
Az: proto.String(c.az),
Payload: proto.String(""), // @todo
ErrorCode: proto.String("com.hailocab.kernel.platform.attemptTimeout"),
ErrorDescription: proto.String(desc),
Duration: proto.Int64(int64(timeout)),
PersistentTrace: proto.Bool(req.TraceShouldPersist()),
})
}
}
// traceRsp decides if we want to trigger a trace event (when processing response) and if so deals with it
func (c *client) traceRsp(req *Request, rsp *Response, err errors.Error, d time.Duration) {
if req.shouldTrace() {
e := &traceproto.Event{
Timestamp: proto.Int64(time.Now().UnixNano()),
TraceId: proto.String(req.TraceID()),
Type: traceproto.Event_REP.Enum(),
MessageId: proto.String(req.MessageID()),
From: proto.String(req.From()),
FromEndpoint: proto.String(req.FromEndpoint()),
To: proto.String(fmt.Sprintf("%v.%v", req.Service(), req.Endpoint())),
ParentMessageId: proto.String(req.ParentMessageID()),
Hostname: proto.String(c.hostname),
Az: proto.String(c.az),
Payload: proto.String(""), // @todo
Duration: proto.Int64(int64(d)),
PersistentTrace: proto.Bool(req.TraceShouldPersist()),
}
if err != nil {
e.ErrorCode = proto.String(err.Code())
e.ErrorDescription = proto.String(err.Description())
}
trace.Send(e)
}
}
// callDiscoveryService sends off a request to register or unregister to the discovery service
func (self *discovery) callDiscoveryService(action string, successState bool) error {
log.Infof("[Server] Attempting to %s with the discovery service...", action)
azName, _ := util.GetAwsAZName()
regSize := reg.size()
machineClass := os.Getenv("H2O_MACHINE_CLASS")
endpoints := make([]*register.MultiRequest_Endpoint, regSize)
i := 0
for _, endpoint := range reg.iterate() {
endpoints[i] = ®ister.MultiRequest_Endpoint{
Name: proto.String(endpoint.Name),
Mean: proto.Int32(endpoint.Mean),
Upper95: proto.Int32(endpoint.Upper95),
Subscribe: proto.String(endpoint.Subscribe),
}
i++
}
service := &dscShared.Service{
Name: proto.String(Name),
Description: proto.String(Description),
Version: proto.Uint64(Version),
Source: proto.String(Source),
OwnerEmail: proto.String(OwnerEmail),
OwnerMobile: proto.String(OwnerMobile),
OwnerTeam: proto.String(OwnerTeam),
}
request, err := ScopedRequest(
"com.hailocab.kernel.discovery",
action,
®ister.MultiRequest{
InstanceId: proto.String(InstanceID),
Hostname: proto.String(self.hostname),
MachineClass: proto.String(machineClass),
AzName: proto.String(azName),
Service: service,
Endpoints: endpoints,
},
)
if err != nil {
log.Warnf("[Server] Failed to build request when %sing services", action)
return err
}
// explicitly define timeout, since we're happy to wait
clientOptions := client.Options{"retries": 0, "timeout": 5 * time.Second}
rsp := ®ister.Response{}
if err := client.Req(request, rsp, clientOptions); err != nil {
log.Warnf("[Server] Failed to %s services: %v", action, err)
return err
}
// ok -- all done!
self.connected = successState
log.Infof("[Server] Successfully %sed with the hive mind!", action)
return nil
}
// loadedConfigHandler handles inbound requests to `loadedconfig` endpoint
func loadedConfigHandler(req *Request) (proto.Message, errors.Error) {
configJson := string(config.Raw())
return &loadedconfigproto.Response{
Config: proto.String(configJson),
}, nil
}
// TestAuthHappyCaseValid tests when things work, and when the credentials are valid
func (suite *sessionRecoverySuite) TestAuthHappyCaseValid() {
t := suite.T()
scope := New().(*realScope)
scope.userCache = newTestCache()
mock := multiclient.NewMock()
stub := &multiclient.Stub{
Service: loginService,
Endpoint: authEndpoint,
Response: &authproto.Response{
SessId: proto.String(testSessId),
Token: proto.String(testToken),
},
}
sessLookupStub := &multiclient.Stub{
Service: loginService,
Endpoint: readSessionEndpoint,
Response: &sessreadproto.Response{
SessId: proto.String(testSessId),
Token: proto.String(testToken),
},
}
mock.Stub(stub).Stub(sessLookupStub)
multiclient.SetCaller(mock.Caller())
testMech, testDeviceType := "h2", "cli"
testUsername, testPassword := "******", "Securez1"
testCreds := map[string]string{
"username": testUsername,
"password": testPassword,
}
err := scope.Auth(testMech, testDeviceType, testCreds)
if err != nil {
t.Errorf("Unexpected auth error: %v", err)
}
if !scope.IsAuth() {
t.Error("Expecting scope to be IsAuth==true after auth")
}
if !scope.HasTriedAuth() {
t.Error("Expecting scope to have HasTriedAuth()==true after auth")
}
// verify we made correct request(s)
if stub.CountCalls() != 1 {
t.Fatalf("Expecting 1 call to auth; got %v", stub.CountCalls())
}
req := &authproto.Request{}
err = stub.Request(0).Unmarshal(req)
if err != nil {
t.Fatalf("Unexpected error unmarshaling our request: %v", err)
}
if req.GetMech() != testMech {
t.Errorf("Request did not contain our expected mech '%s', got '%s'", testMech, req.GetMech())
}
if req.GetDeviceType() != testDeviceType {
t.Errorf("Request did not contain our expected device type '%s', got '%s'", testDeviceType, req.GetDeviceType())
}
if req.GetUsername() != testUsername {
t.Errorf("Request did not contain our expected username '%s', got '%s'", testUsername, req.GetUsername())
}
if req.GetPassword() != testPassword {
t.Errorf("Request did not contain our expected password '%s', got '%s'", testPassword, req.GetPassword())
}
// verify user returned
u := scope.AuthUser()
if u == nil {
t.Fatal("IsAuthed scope returned nil user")
}
if u.Id != "dave" {
t.Errorf("Expecting user ID 'dave'; got '%s'", u.Id)
}
// clean out scope
scope.Clean()
if scope.IsAuth() {
t.Error("Expecting scope to be IsAuth==false after Clean()")
}
if scope.HasTriedAuth() {
t.Error("Expecting scope to have HasTriedAuth()==false after Clean()")
}
if u := scope.AuthUser(); u != nil {
t.Error("Expecting AuthUser()==nil after Clean()")
}
// auth AGAIN -- we should be calling login service _again_
err = scope.Auth(testMech, testDeviceType, testCreds)
if err != nil {
t.Errorf("Unexpected recover error: %v", err)
}
if !scope.IsAuth() {
t.Error("Expecting scope to be IsAuth=true after recovery")
}
// verify we haven't made _any more_ requests
if stub.CountCalls() != 2 {
t.Fatalf("Expecting 2 call to auth (because it should be called each time); got %v", stub.CountCalls())
}
// session should be pushed into cache
err = scope.RecoverSession(testSessId)
if err != nil {
t.Errorf("Unexpected recover error: %v", err)
}
if sessLookupStub.CountCalls() != 0 {
t.Fatalf("Expecting 0 call to readsession (because it should be cached); got %v", stub.CountCalls())
}
}
// TestSignOutHappy tests signout when it works
func (suite *sessionRecoverySuite) TestSignOutHappy() {
t := suite.T()
scope := New().(*realScope)
scope.userCache = newTestCache()
mock := multiclient.NewMock()
stub := &multiclient.Stub{
Service: loginService,
Endpoint: deleteSessionEndpoint,
Response: &sessdelproto.Response{},
}
readSessStub := &multiclient.Stub{
Service: loginService,
Endpoint: readSessionEndpoint,
Response: &sessreadproto.Response{
SessId: proto.String(testSessId),
Token: proto.String(testToken),
},
}
mock.Stub(stub).Stub(readSessStub)
multiclient.SetCaller(mock.Caller())
// need to recover first
err := scope.RecoverSession(testSessId)
if err != nil || !scope.IsAuth() {
t.Errorf("Unexpected recover error, or not Authed as expected: %v", err)
}
// now signout
err = scope.SignOut(scope.AuthUser())
if err != nil {
t.Errorf("Unexpected signout error: %v", err)
}
// verify we made correct request(s)
if stub.CountCalls() != 1 {
t.Fatalf("Expecting 1 call to deletesession; got %v", stub.CountCalls())
}
req := &sessdelproto.Request{}
err = stub.Request(0).Unmarshal(req)
if err != nil {
t.Fatalf("Unexpected error unmarshaling our request: %v", err)
}
if req.GetSessId() != testSessId {
t.Errorf("Request did not contain our expected sessId '%s', got '%s'", testSessId, req.GetSessId())
}
if scope.IsAuth() {
t.Error("Expecting scope to be IsAuth==false after SignOut()")
}
if scope.HasTriedAuth() {
t.Error("Expecting scope to have HasTriedAuth()==false after SignOut()")
}
if u := scope.AuthUser(); u != nil {
t.Error("Expecting AuthUser()==nil after SignOut()")
}
// make sure we have purged cache
err = scope.RecoverSession(testSessId)
if err != nil {
t.Errorf("Expecting error when recovering session again")
}
if readSessStub.CountCalls() != 2 {
t.Error("Expecting 2 calls to readsession - sincee we should have purged the cache after SignOut()")
}
}
// traceOut traces a request outbound from a service handler
func traceOut(req *Request, msg proto.Message, err perrors.Error, d time.Duration) {
if req.shouldTrace() {
e := &traceproto.Event{
Timestamp: proto.Int64(time.Now().UnixNano()),
TraceId: proto.String(req.TraceID()),
Type: traceproto.Event_OUT.Enum(),
MessageId: proto.String(req.MessageID()),
ParentMessageId: proto.String(req.ParentMessageID()),
From: proto.String(req.From()),
To: proto.String(fmt.Sprintf("%v.%v", req.Service(), req.Endpoint())),
Hostname: proto.String(hostname),
Az: proto.String(az),
Payload: proto.String(""), // @todo
HandlerInstanceId: proto.String(InstanceID),
Duration: proto.Int64(int64(d)),
PersistentTrace: proto.Bool(req.TraceShouldPersist()),
}
if err != nil {
e.ErrorCode = proto.String(err.Code())
e.ErrorDescription = proto.String(err.Description())
}
trace.Send(e)
}
}