func (s *stats) runtime() *pstats.RuntimeStats {
return &pstats.RuntimeStats{
HeapInUse: proto.Uint64(uint64(getGaugeVal(s.registry, "runtime.MemStats.HeapInuse"))),
HeapTotal: proto.Uint64(uint64(getGaugeVal(s.registry, "runtime.MemStats.TotalAlloc"))),
HeapReleased: proto.Uint64(uint64(getGaugeVal(s.registry, "runtime.MemStats.HeapReleased"))),
LastGCDuration: proto.Uint64(uint64(getGaugeVal(s.registry, "local.GCStats.LastGCDuration"))),
NumGC: proto.Uint32(uint32(getGaugeVal(s.registry, "debug.GCStats.NumGC"))),
NumGoRoutines: proto.Uint32(uint32(getGaugeVal(s.registry, "runtime.NumGoroutine"))),
}
}
// publishFailure publishes a failure/panic event to be monitored.
func publishFailure(r interface{}) {
var p string
switch r.(type) {
case string:
p = r.(string)
case error:
p = fmt.Sprintf("%v", r.(error))
default:
p = "Unknown panic"
}
b := make([]byte, 1024)
runtime.Stack(b, true)
if err := client.Pub("com.hailocab.monitor.failure", &fproto.Failure{
ServiceName: proto.String(Name),
ServiceVersion: proto.Uint64(Version),
AzName: proto.String(az),
Hostname: proto.String(hostname),
InstanceId: proto.String(InstanceID),
Timestamp: proto.Int64(time.Now().Unix()),
Uptime: proto.Int64(int64(time.Since(serviceStarted).Seconds())),
Type: proto.String("PANIC"),
Reason: proto.String(p),
Stack: proto.String(string(b)),
}); err != nil {
log.Errorf("[Server] Failed to publish failure event: %v", err)
}
}
func healthCheckSampleToProto(hc *HealthCheck, sample *Sample) *hcproto.HealthCheck {
return &hcproto.HealthCheck{
Timestamp: proto.Int64(sample.At.Unix()),
HealthCheckId: proto.String(hc.Id),
ServiceName: proto.String(hc.ServiceName),
ServiceVersion: proto.Uint64(hc.ServiceVersion),
Hostname: proto.String(hc.Hostname),
InstanceId: proto.String(hc.InstanceId),
IsHealthy: proto.Bool(sample.IsHealthy),
ErrorDescription: proto.String(sample.ErrorDescription),
Measurements: mapToProto(sample.Measurements),
Priority: hcproto.HealthCheck_Priority(hc.Priority).Enum(),
}
}
func TestResponder(t *testing.T) {
stub := &Stub{
Service: mockFooService,
Endpoint: mockHealthEndpoint,
Responder: func(invocation int, req *client.Request) (proto.Message, errors.Error) {
if invocation == 1 {
return &hcproto.Response{
Healthchecks: []*hcproto.HealthCheck{
&hcproto.HealthCheck{
Timestamp: proto.Int64(1403629015),
ServiceName: proto.String("foo"),
ServiceVersion: proto.Uint64(1403629015),
Hostname: proto.String("localhost"),
InstanceId: proto.String("foobar"),
HealthCheckId: proto.String("boom"),
IsHealthy: proto.Bool(true),
},
},
}, nil
}
return nil, errors.InternalServerError("only.one.allowed", "First call only works")
},
}
mock := NewMock().Stub(stub)
caller := mock.Caller()
req, _ := client.NewRequest(mockFooService, mockHealthEndpoint, &hcproto.Request{})
rsp := &hcproto.Response{}
e := caller(req, rsp)
assert.Nil(t, e,
"Expecting our mocked call to be intercepted and stubbed response returned, got err: %v", e)
assert.Len(t, rsp.GetHealthchecks(), 1,
"Response does not contain our mocked content: no healthchecks")
// now repeat, and we SHOULD get an error
e = caller(req, rsp)
assert.NotNil(t, e,
"Expecting our mocked call to be intercepted and error response returned on 2nd call")
assert.Equal(t, e.Code(), "only.one.allowed",
"Expecting code 'only.one.allowed', got '%s'", e.Code())
}
// get returns a snapshot of platform stats
func (s *stats) get(status string) *pstats.PlatformStats {
rusageStats := s.rusage()
runtimeStats := s.runtime()
endpointStats := s.endpoints()
return &pstats.PlatformStats{
ServiceName: proto.String(ServiceName),
ServiceVersion: proto.Uint64(ServiceVersion),
ServiceType: proto.String(ServiceType),
AzName: proto.String(AzName),
Hostname: proto.String(hostname),
InstanceId: proto.String(InstanceID),
Status: proto.String(status),
Timestamp: proto.Int64(time.Now().Unix()),
Uptime: proto.Int64(int64(time.Since(s.startTime).Seconds())),
Rusage: rusageStats,
Runtime: runtimeStats,
Endpoints: endpointStats,
}
}
func TestMockCallerPopulatesResponse(t *testing.T) {
req, _ := client.NewRequest(mockFooService, mockHealthEndpoint, &hcproto.Request{})
stub := &Stub{
Service: mockFooService,
Endpoint: mockHealthEndpoint,
Response: &hcproto.Response{
Healthchecks: []*hcproto.HealthCheck{
&hcproto.HealthCheck{
Timestamp: proto.Int64(1403629015),
ServiceName: proto.String("foo"),
ServiceVersion: proto.Uint64(1403629015),
Hostname: proto.String("localhost"),
InstanceId: proto.String("foobar"),
HealthCheckId: proto.String("boom"),
IsHealthy: proto.Bool(true),
},
},
},
}
mock := NewMock().Stub(stub)
caller := mock.Caller()
rsp := &hcproto.Response{}
e := caller(req, rsp)
assert.Nil(t, e,
"Expecting our mocked call to be intercepted and stubbed response returned, got err: %v", e)
// ensure stub has what we expect
assert.Len(t, stub.matched, 1,
"Expecting 1 match payload to be stored after execution")
assert.Equal(t, stub.CountCalls(), 1, "CountCalls should return 1 too")
assert.Len(t, rsp.GetHealthchecks(), 1,
"Response does not contain our mocked content: no healthchecks")
}
// callDiscoveryService sends off a request to register or unregister to the discovery service
func (self *discovery) callDiscoveryService(action string, successState bool) error {
log.Infof("[Server] Attempting to %s with the discovery service...", action)
azName, _ := util.GetAwsAZName()
regSize := reg.size()
machineClass := os.Getenv("H2O_MACHINE_CLASS")
endpoints := make([]*register.MultiRequest_Endpoint, regSize)
i := 0
for _, endpoint := range reg.iterate() {
endpoints[i] = ®ister.MultiRequest_Endpoint{
Name: proto.String(endpoint.Name),
Mean: proto.Int32(endpoint.Mean),
Upper95: proto.Int32(endpoint.Upper95),
Subscribe: proto.String(endpoint.Subscribe),
}
i++
}
service := &dscShared.Service{
Name: proto.String(Name),
Description: proto.String(Description),
Version: proto.Uint64(Version),
Source: proto.String(Source),
OwnerEmail: proto.String(OwnerEmail),
OwnerMobile: proto.String(OwnerMobile),
OwnerTeam: proto.String(OwnerTeam),
}
request, err := ScopedRequest(
"com.hailocab.kernel.discovery",
action,
®ister.MultiRequest{
InstanceId: proto.String(InstanceID),
Hostname: proto.String(self.hostname),
MachineClass: proto.String(machineClass),
AzName: proto.String(azName),
Service: service,
Endpoints: endpoints,
},
)
if err != nil {
log.Warnf("[Server] Failed to build request when %sing services", action)
return err
}
// explicitly define timeout, since we're happy to wait
clientOptions := client.Options{"retries": 0, "timeout": 5 * time.Second}
rsp := ®ister.Response{}
if err := client.Req(request, rsp, clientOptions); err != nil {
log.Warnf("[Server] Failed to %s services: %v", action, err)
return err
}
// ok -- all done!
self.connected = successState
log.Infof("[Server] Successfully %sed with the hive mind!", action)
return nil
}