func (d *dataAccess) GetUnknownAndUnevalAlertKeys(alert string) ([]models.AlertKey, []models.AlertKey, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetUnknownAndUnevalAlertKeys"})()
conn := d.GetConnection()
defer conn.Close()
unknownS, err := redis.Strings(conn.Do("SMEMBERS", statesUnknownKey(alert)))
if err != nil {
return nil, nil, slog.Wrap(err)
}
unknown := make([]models.AlertKey, len(unknownS))
for i, u := range unknownS {
unknown[i] = models.AlertKey(u)
}
unEvals, err := redis.Strings(conn.Do("SMEMBERS", statesUnevalKey(alert)))
if err != nil {
return nil, nil, slog.Wrap(err)
}
unevals := make([]models.AlertKey, len(unEvals))
for i, u := range unEvals {
unevals[i] = models.AlertKey(u)
}
return unknown, unevals, nil
}
func (d *dataAccess) BackupLastInfos(m map[string]map[string]*LastInfo) error {
conn := d.Get()
defer conn.Close()
dat, err := util.MarshalGzipJson(m)
if err != nil {
return slog.Wrap(err)
}
_, err = conn.Do("SET", "search:last", dat)
return slog.Wrap(err)
}
func (d *dataAccess) InsertNotification(ak models.AlertKey, notification string, dueAt time.Time) error {
conn := d.Get()
defer conn.Close()
_, err := conn.Do("ZADD", pendingNotificationsKey, dueAt.UTC().Unix(), fmt.Sprintf("%s:%s", ak, notification))
if err != nil {
return slog.Wrap(err)
}
_, err = conn.Do("SADD", notsByAlertKeyKey(ak), notification)
return slog.Wrap(err)
}
func (d *dataAccess) getIncident(incidentId int64, conn redis.Conn) (*models.IncidentState, error) {
b, err := redis.Bytes(conn.Do("GET", incidentStateKey(incidentId)))
if err != nil {
return nil, slog.Wrap(err)
}
state := &models.IncidentState{}
if err = json.Unmarshal(b, state); err != nil {
return nil, slog.Wrap(err)
}
return state, nil
}
func (d *dataAccess) BackupLastInfos(m map[string]map[string]*LastInfo) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "BackupLast"})()
conn := d.GetConnection()
defer conn.Close()
dat, err := util.MarshalGzipJson(m)
if err != nil {
return slog.Wrap(err)
}
_, err = conn.Do("SET", "search:last", dat)
return slog.Wrap(err)
}
func (d *dataAccess) InsertNotification(ak models.AlertKey, notification string, dueAt time.Time) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "InsertNotification"})()
conn := d.GetConnection()
defer conn.Close()
_, err := conn.Do("ZADD", pendingNotificationsKey, dueAt.UTC().Unix(), fmt.Sprintf("%s:%s", ak, notification))
if err != nil {
return slog.Wrap(err)
}
_, err = conn.Do("SADD", notsByAlertKeyKey(ak), notification)
return slog.Wrap(err)
}
func (d *dataAccess) GetTempConfig(hash string) (string, error) {
conn := d.Get()
defer conn.Close()
key := "tempConfig:" + hash
dat, err := redis.String(conn.Do("GET", key))
if err != nil {
return "", slog.Wrap(err)
}
_, err = conn.Do("EXPIRE", key, configLifetime)
return dat, slog.Wrap(err)
}
func (d *dataAccess) SaveTempConfig(text string) (string, error) {
conn := d.Get()
defer conn.Close()
sig := md5.Sum([]byte(text))
b64 := base64.StdEncoding.EncodeToString(sig[0:8])
if d.isRedis {
_, err := conn.Do("SET", "tempConfig:"+b64, text, "EX", configLifetime)
return b64, slog.Wrap(err)
}
_, err := conn.Do("SETEX", "tempConfig:"+b64, configLifetime, text)
return b64, slog.Wrap(err)
}
func (d *dataAccess) GetTempConfig(hash string) (string, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetTempConfig"})()
conn := d.GetConnection()
defer conn.Close()
key := "tempConfig:" + hash
dat, err := redis.String(conn.Do("GET", key))
if err != nil {
return "", slog.Wrap(err)
}
_, err = conn.Do("EXPIRE", key, configLifetime)
return dat, slog.Wrap(err)
}
func c_redis_counters(server string, db int) (opentsdb.MultiDataPoint, error) {
var md opentsdb.MultiDataPoint
conn, err := redis.Dial("tcp", server, redis.DialDatabase(db))
if err != nil {
return md, slog.Wrap(err)
}
defer conn.Close()
//do a dance to detect proper hscan command for ledis or redis
hscanCmd := "XHSCAN"
info, err := redis.String(conn.Do("info", "server"))
if err != nil {
return md, slog.Wrap(err)
}
if strings.Contains(info, "redis_version") {
hscanCmd = "HSCAN"
}
cursor := "0"
for {
vals, err := redis.Values(conn.Do(hscanCmd, collect.RedisCountersKey, cursor))
if err != nil {
return md, slog.Wrap(err)
}
if len(vals) != 2 {
return md, fmt.Errorf("Unexpected number of values")
}
cursor, err = redis.String(vals[0], nil)
if err != nil {
return md, slog.Wrap(err)
}
pairs, err := redis.StringMap(vals[1], nil)
if err != nil {
return md, slog.Wrap(err)
}
for key, val := range pairs {
ak := models.AlertKey(key)
v, err := strconv.Atoi(val)
if err != nil {
slog.Errorf("Invalid counter value: %s", val)
continue
}
Add(&md, ak.Name(), v, ak.Group(), metadata.Counter, metadata.Count, "")
}
if cursor == "" || cursor == "0" {
break
}
}
return md, nil
}
func (d *dataAccess) SaveTempConfig(text string) (string, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "SaveTempConfig"})()
conn := d.GetConnection()
defer conn.Close()
sig := md5.Sum([]byte(text))
b64 := base64.StdEncoding.EncodeToString(sig[0:8])
if d.isRedis {
_, err := conn.Do("SET", "tempConfig:"+b64, text, "EX", configLifetime)
return b64, slog.Wrap(err)
}
_, err := conn.Do("SETEX", "tempConfig:"+b64, configLifetime, text)
return b64, slog.Wrap(err)
}
func (d *dataAccess) getLatestIncident(ak models.AlertKey, conn redis.Conn) (*models.IncidentState, error) {
id, err := redis.Int64(conn.Do("LINDEX", incidentsForAlertKeyKey(ak), 0))
if err != nil {
if err == redis.ErrNil {
return nil, nil
}
return nil, slog.Wrap(err)
}
inc, err := d.getIncident(id, conn)
if err != nil {
return nil, slog.Wrap(err)
}
return inc, nil
}
func int64s(reply interface{}, err error) ([]int64, error) {
if err != nil {
return nil, slog.Wrap(err)
}
ints := []int64{}
values, err := redis.Values(reply, err)
if err != nil {
return ints, slog.Wrap(err)
}
if err := redis.ScanSlice(values, &ints); err != nil {
return ints, slog.Wrap(err)
}
return ints, nil
}
func (d *dataAccess) transact(conn redis.Conn, f func() error) error {
if !d.isRedis {
return f()
}
if _, err := conn.Do("MULTI"); err != nil {
return slog.Wrap(err)
}
if err := f(); err != nil {
return slog.Wrap(err)
}
if _, err := conn.Do("EXEC"); err != nil {
return slog.Wrap(err)
}
return nil
}
func (d *dataAccess) LoadLastInfos() (map[string]map[string]*LastInfo, error) {
conn := d.Get()
defer conn.Close()
b, err := redis.Bytes(conn.Do("GET", "search:last"))
if err != nil {
return nil, slog.Wrap(err)
}
var m map[string]map[string]*LastInfo
err = util.UnmarshalGzipJson(b, &m)
if err != nil {
return nil, slog.Wrap(err)
}
return m, nil
}
func (d *dataAccess) AddMetric(metric string, time int64) error {
conn := d.Get()
defer conn.Close()
_, err := conn.Do("HSET", searchAllMetricsKey, metric, time)
return slog.Wrap(err)
}
func c_mssql_statistics(svc_dst []Win32_Service) (opentsdb.MultiDataPoint, error) {
var md opentsdb.MultiDataPoint
for _, w := range svc_dst {
var dst []Win32_PerfRawData_MSSQLSERVER_SQLServerSQLStatistics
q := wmi.CreateQuery(&dst, `WHERE Name <> '_Total'`)
label := "mssqlserver"
if w.Name != `MSSQLSERVER` {
q = instanceWMIQuery(w.Name, q)
label = strings.ToLower(w.Name[6:len(w.Name)])
}
err := queryWmi(q, &dst)
if err != nil {
return nil, slog.Wrap(err)
}
for _, v := range dst {
tags := opentsdb.TagSet{"instance": label}
Add(&md, "mssql.autoparam_attempts", v.AutoParamAttemptsPersec, tags, metadata.Counter, metadata.PerSecond, descMSSQLAutoParamAttemptsPersec)
Add(&md, "mssql.autoparam_failed", v.FailedAutoParamsPersec, tags, metadata.Counter, metadata.PerSecond, descMSSQLFailedAutoParamsPersec)
Add(&md, "mssql.autoparam_forced", v.ForcedParameterizationsPersec, tags, metadata.Counter, metadata.PerSecond, descMSSQLForcedParameterizationsPersec)
Add(&md, "mssql.autoparam_safe", v.SafeAutoParamsPersec, tags, metadata.Counter, metadata.PerSecond, descMSSQLSafeAutoParamsPersec)
Add(&md, "mssql.autoparam_unsafe", v.UnsafeAutoParamsPersec, tags, metadata.Counter, metadata.PerSecond, descMSSQLUnsafeAutoParamsPersec)
Add(&md, "mssql.batches", v.BatchRequestsPersec, tags, metadata.Counter, metadata.PerSecond, descMSSQLBatchRequestsPersec)
Add(&md, "mssql.guided_plans", v.GuidedplanexecutionsPersec, tags, metadata.Counter, metadata.PerSecond, descMSSQLGuidedplanexecutionsPersec)
Add(&md, "mssql.misguided_plans", v.MisguidedplanexecutionsPersec, tags, metadata.Counter, metadata.PerSecond, descMSSQLMisguidedplanexecutionsPersec)
Add(&md, "mssql.compilations", v.SQLCompilationsPersec, tags, metadata.Counter, metadata.PerSecond, descMSSQLSQLCompilationsPersec)
Add(&md, "mssql.recompilations", v.SQLReCompilationsPersec, tags, metadata.Counter, metadata.PerSecond, descMSSQLSQLReCompilationsPersec)
}
}
return md, nil
}
func c_mssql_locks(svc_dst []Win32_Service) (opentsdb.MultiDataPoint, error) {
var md opentsdb.MultiDataPoint
for _, w := range svc_dst {
var dst []Win32_PerfRawData_MSSQLSERVER_SQLServerLocks
q := wmi.CreateQuery(&dst, `WHERE Name = 'Page' OR Name = 'Extent' OR Name = 'Object' or Name = 'Database'`)
label := "mssqlserver"
if w.Name != `MSSQLSERVER` {
q = instanceWMIQuery(w.Name, q)
label = strings.ToLower(w.Name[6:len(w.Name)])
}
err := queryWmi(q, &dst)
if err != nil {
return nil, slog.Wrap(err)
}
for _, v := range dst {
tags := opentsdb.TagSet{"instance": label, "type": v.Name}
Add(&md, "mssql.lock_wait_time", v.AverageWaitTimems, tags, metadata.Counter, metadata.MilliSecond, descMSSQLAverageWaitTimems)
Add(&md, "mssql.lock_requests", v.LockRequestsPersec, tags, metadata.Counter, metadata.PerSecond, descMSSQLLockRequestsPersec)
Add(&md, "mssql.lock_timeouts", v.LockTimeoutsPersec, tags, metadata.Counter, metadata.PerSecond, descMSSQLLockTimeoutsPersec)
Add(&md, "mssql.lock_timeouts0", v.LockTimeoutstimeout0Persec, tags, metadata.Counter, metadata.PerSecond, descMSSQLLockTimeoutstimeout0Persec)
Add(&md, "mssql.lock_waits", v.LockWaitsPersec, tags, metadata.Counter, metadata.PerSecond, descMSSQLLockWaitsPersec)
Add(&md, "mssql.deadlocks", v.NumberofDeadlocksPersec, tags, metadata.Counter, metadata.PerSecond, descMSSQLNumberofDeadlocksPersec)
}
}
return md, nil
}
func (d *dataAccess) LoadLastInfos() (map[string]map[string]*LastInfo, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "LoadLast"})()
conn := d.GetConnection()
defer conn.Close()
b, err := redis.Bytes(conn.Do("GET", "search:last"))
if err != nil {
return nil, slog.Wrap(err)
}
var m map[string]map[string]*LastInfo
err = util.UnmarshalGzipJson(b, &m)
if err != nil {
return nil, slog.Wrap(err)
}
return m, nil
}
func c_mssql_replica_server() (opentsdb.MultiDataPoint, error) {
var dst []Win32_PerfRawData_MSSQLSERVER_SQLServerAvailabilityReplica
if err := queryWmi(sqlAGQuery, &dst); err != nil {
return nil, slog.Wrap(err)
}
var md opentsdb.MultiDataPoint
for _, v := range dst {
//split name into AvailibilityGroup and Destination. Name is in 'Group:Destination' format
s := strings.Split(v.Name, ":")
if len(s) != 2 {
return nil, fmt.Errorf("Invalid Availibility Group Name: '%s'", v.Name)
}
destination := strings.ToLower(s[1])
//see http://technet.microsoft.com/en-us/library/ff878472(v=sql.110).aspx
//also https://livedemo.customers.na.apm.ibmserviceengage.com/help/index.jsp?topic=%2Fcom.ibm.koq.doc%2Fattr_koqadbst.htm
Add(&md, "mssql.replica.bytes_ag", v.BytesReceivedfromReplicaPersec, opentsdb.TagSet{"group": s[0], "destination": destination, "type": "received"}, metadata.Counter, metadata.BytesPerSecond, descMSSQLReplicaBytesReceivedfromReplicaPersec)
Add(&md, "mssql.replica.bytes_ag", v.BytesSenttoReplicaPersec, opentsdb.TagSet{"group": s[0], "destination": destination, "type": "sent_replica"}, metadata.Counter, metadata.BytesPerSecond, descMSSQLReplicaBytesSenttoReplicaPersec)
Add(&md, "mssql.replica.bytes_ag", v.BytesSenttoTransportPersec, opentsdb.TagSet{"group": s[0], "destination": destination, "type": "sent_transport"}, metadata.Counter, metadata.BytesPerSecond, descMSSQLReplicaBytesSenttoTransportPersec)
Add(&md, "mssql.replica.delay_flow", v.FlowControlTimemsPersec, opentsdb.TagSet{"group": s[0], "destination": destination}, metadata.Counter, metadata.MilliSecond, descMSSQLReplicaFlowControlTimemsPersec)
Add(&md, "mssql.replica.messages", v.FlowControlPersec, opentsdb.TagSet{"group": s[0], "destination": destination, "type": "flow_control"}, metadata.Counter, metadata.PerSecond, descMSSQLReplicaFlowControlPersec)
Add(&md, "mssql.replica.messages", v.ReceivesfromReplicaPersec, opentsdb.TagSet{"group": s[0], "destination": destination, "type": "received"}, metadata.Counter, metadata.PerSecond, descMSSQLReplicaReceivesfromReplicaPersec)
Add(&md, "mssql.replica.messages", v.ResentMessagesPersec, opentsdb.TagSet{"group": s[0], "destination": destination, "type": "resent"}, metadata.Counter, metadata.PerSecond, descMSSQLReplicaResentMessagesPersec)
Add(&md, "mssql.replica.messages", v.SendstoReplicaPersec, opentsdb.TagSet{"group": s[0], "destination": destination, "type": "sent_replica"}, metadata.Counter, metadata.PerSecond, descMSSQLReplicaSendstoReplicaPersec)
Add(&md, "mssql.replica.messages", v.SendstoTransportPersec, opentsdb.TagSet{"group": s[0], "destination": destination, "type": "sent_transport"}, metadata.Counter, metadata.PerSecond, descMSSQLReplicaSendstoTransportPersec)
}
return md, nil
}
func (d *dataAccess) ClearNotificationsBefore(t time.Time) error {
conn := d.Get()
defer conn.Close()
_, err := conn.Do("ZREMRANGEBYSCORE", pendingNotificationsKey, 0, t.UTC().Unix())
return slog.Wrap(err)
}
func c_mssql_general(svc_dst []Win32_Service) (opentsdb.MultiDataPoint, error) {
var md opentsdb.MultiDataPoint
for _, w := range svc_dst {
var dst []Win32_PerfRawData_MSSQLSERVER_SQLServerGeneralStatistics
q := wmi.CreateQuery(&dst, `WHERE Name <> '_Total'`)
label := "mssqlserver"
if w.Name != `MSSQLSERVER` {
q = instanceWMIQuery(w.Name, q)
label = strings.ToLower(w.Name[6:len(w.Name)])
}
if err := queryWmi(q, &dst); err != nil {
return nil, slog.Wrap(err)
}
for _, v := range dst {
tags := opentsdb.TagSet{"instance": label}
Add(&md, "mssql.user_connections", v.UserConnections, tags, metadata.Gauge, metadata.Count, descMSSQLUserConnections)
Add(&md, "mssql.connection_resets", v.ConnectionResetPersec, tags, metadata.Counter, metadata.PerSecond, descMSSQLConnectionResetPersec)
Add(&md, "mssql.logins", v.LoginsPersec, tags, metadata.Counter, metadata.PerSecond, descMSSQLLoginsPersec)
Add(&md, "mssql.logouts", v.LogoutsPersec, tags, metadata.Counter, metadata.PerSecond, descMSSQLLogoutsPersec)
Add(&md, "mssql.mars_deadlocks", v.MarsDeadlocks, tags, metadata.Counter, metadata.Count, descMSSQLMarsDeadlocks)
Add(&md, "mssql.proc_blocked", v.Processesblocked, tags, metadata.Gauge, metadata.Count, descMSSQLProcessesblocked)
Add(&md, "mssql.temptables_created", v.TempTablesCreationRate, tags, metadata.Counter, metadata.PerSecond, descMSSQLTempTablesCreationRate)
Add(&md, "mssql.temptables_to_destroy", v.TempTablesForDestruction, tags, metadata.Gauge, metadata.Count, descMSSQLTempTablesForDestruction)
Add(&md, "mssql.transactions_total", v.Transactions, tags, metadata.Gauge, metadata.Count, descMSSQLTransactions)
}
}
return md, nil
}
func (d *dataAccess) AddMetricTagSet(metric, tagSet string, time int64) error {
conn := d.Get()
defer conn.Close()
_, err := conn.Do("HSET", searchMetricTagSetKey(metric), tagSet, time)
return slog.Wrap(err)
}
func (d *dataAccess) AddTagKeyForMetric(metric, tagK string, time int64) error {
conn := d.Get()
defer conn.Close()
_, err := conn.Do("HSET", searchTagkKey(metric), tagK, time)
return slog.Wrap(err)
}
func (d *dataAccess) AddMetricForTag(tagK, tagV, metric string, time int64) error {
conn := d.Get()
defer conn.Close()
_, err := conn.Do("HSET", searchMetricKey(tagK, tagV), metric, time)
return slog.Wrap(err)
}
func (d *dataAccess) TouchAlertKey(ak models.AlertKey, t time.Time) error {
conn := d.Get()
defer conn.Close()
_, err := conn.Do("ZADD", statesLastTouchedKey(ak.Name()), t.UTC().Unix(), string(ak))
return slog.Wrap(err)
}
func (d *dataAccess) ClearNotificationsBefore(t time.Time) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "ClearNotificationsBefore"})()
conn := d.GetConnection()
defer conn.Close()
_, err := conn.Do("ZREMRANGEBYSCORE", pendingNotificationsKey, 0, t.UTC().Unix())
return slog.Wrap(err)
}
func (d *dataAccess) TouchAlertKey(ak models.AlertKey, t time.Time) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "TouchAlertKey"})()
conn := d.GetConnection()
defer conn.Close()
_, err := conn.Do("ZADD", statesLastTouchedKey(ak.Name()), t.UTC().Unix(), string(ak))
return slog.Wrap(err)
}
func (d *dataAccess) AddMetricForTag(tagK, tagV, metric string, time int64) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "AddMetricForTag"})()
conn := d.GetConnection()
defer conn.Close()
_, err := conn.Do("HSET", searchMetricKey(tagK, tagV), metric, time)
return slog.Wrap(err)
}
func (d *dataAccess) AddTagKeyForMetric(metric, tagK string, time int64) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "AddTagKeyForMetric"})()
conn := d.GetConnection()
defer conn.Close()
_, err := conn.Do("HSET", searchTagkKey(metric), tagK, time)
return slog.Wrap(err)
}