コード例 #1
0
ファイル: database.go プロジェクト: MichaelS11/bosun
func DatabaseConnect(c conf.Database) (*sql.DB, error) {
	var db *sql.DB
	var err error

	if c.Type == "mysql" {
		if c.Password == "" {
			db, err = sql.Open("mysql", c.Username+"@"+c.Protocol+"("+c.Address+":"+strconv.Itoa(c.Port)+")/")
		} else {
			db, err = sql.Open("mysql", c.Username+":"+c.Password+"@"+c.Protocol+"("+c.Address+":"+strconv.Itoa(c.Port)+")/")
		}
	}

	if err != nil {
		slog.Errorf("%v: %v: %v: %v", "Database", c.Type, "connect open error", err)
		return nil, fmt.Errorf("%v: %v", "connect open error", err)
	}

	// check if can connect to database, if fail, check again every minute until connected
	for {
		err = db.Ping()
		if err == nil {
			break
		}
		slog.Errorf("%v: %v: %v: %v", "Database", c.Type, "connect ping error", err)
		next := time.After(time.Minute)
		select {
		case <-next:
		case <-ContinuousCollectorVars.quit:
			db.Close()
			return nil, fmt.Errorf("connect while quitting")
		}
	}

	return db, nil
}
コード例 #2
0
ファイル: command.go プロジェクト: nicollet/bosun
// Command executes the named program with the given arguments. If it does not
// exit within timeout, it is sent SIGINT (if supported by Go). After
// another timeout, it is killed.
func Command(timeout time.Duration, stdin io.Reader, name string, arg ...string) (io.Reader, error) {
	if _, err := exec.LookPath(name); err != nil {
		return nil, ErrPath
	}
	if Debug {
		slog.Infof("executing command: %v %v", name, arg)
	}
	c := exec.Command(name, arg...)
	b := &bytes.Buffer{}
	c.Stdout = b
	c.Stdin = stdin
	if err := c.Start(); err != nil {
		return nil, err
	}
	timedOut := false
	intTimer := time.AfterFunc(timeout, func() {
		slog.Errorf("Process taking too long. Interrupting: %s %s", name, strings.Join(arg, " "))
		c.Process.Signal(os.Interrupt)
		timedOut = true
	})
	killTimer := time.AfterFunc(timeout*2, func() {
		slog.Errorf("Process taking too long. Killing: %s %s", name, strings.Join(arg, " "))
		c.Process.Signal(os.Kill)
		timedOut = true
	})
	err := c.Wait()
	intTimer.Stop()
	killTimer.Stop()
	if timedOut {
		return nil, ErrTimeout
	}
	return b, err
}
コード例 #3
0
ファイル: bolt.go プロジェクト: youngl98/bosun
func (s *Schedule) save() {
	if s.db == nil {
		return
	}
	s.Lock("Save")
	store := map[string]interface{}{
		dbMetric:        s.Search.Read.Metric,
		dbTagk:          s.Search.Read.Tagk,
		dbTagv:          s.Search.Read.Tagv,
		dbMetricTags:    s.Search.Read.MetricTags,
		dbNotifications: s.Notifications,
		dbSilence:       s.Silence,
		dbStatus:        s.status,
		dbMetadata:      s.Metadata,
		dbIncidents:     s.Incidents,
	}
	tostore := make(map[string][]byte)
	for name, data := range store {
		f := new(bytes.Buffer)
		gz := gzip.NewWriter(f)
		cw := &counterWriter{w: gz}
		enc := gob.NewEncoder(cw)
		if err := enc.Encode(data); err != nil {
			slog.Errorf("error saving %s: %v", name, err)
			s.Unlock()
			return
		}
		if err := gz.Flush(); err != nil {
			slog.Errorf("gzip flush error saving %s: %v", name, err)
		}
		if err := gz.Close(); err != nil {
			slog.Errorf("gzip close error saving %s: %v", name, err)
		}
		tostore[name] = f.Bytes()
		slog.Infof("wrote %s: %v", name, conf.ByteSize(cw.written))
		collect.Put("statefile.size", opentsdb.TagSet{"object": name}, cw.written)
	}
	s.Unlock()
	err := s.db.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists([]byte(dbBucket))
		if err != nil {
			return err
		}
		for name, data := range tostore {
			if err := b.Put([]byte(name), data); err != nil {
				return err
			}
		}
		return nil
	})
	if err != nil {
		slog.Errorf("save db update error: %v", err)
		return
	}
	fi, err := os.Stat(s.Conf.StateFile)
	if err == nil {
		collect.Put("statefile.size", opentsdb.TagSet{"object": "total"}, fi.Size())
	}
	slog.Infoln("save to db complete")
}
コード例 #4
0
ファイル: elasticsearch.go プロジェクト: eswdd/bosun
// structProcessor.add() takes in a metric name prefix, an arbitrary struct, and a tagset.
// The processor recurses through the struct and builds metrics. The field tags direct how
// the field should be processed, as well as the metadata for the resulting metric.
//
// The field tags used are described as follows:
//
// version: typically set to '1' or '2'.
//	This is compared against the elastic cluster version. If the version from the tag
//      does not match the version in production, the metric will not be sent for this field.
//
// exclude:
//      If this tag is set to 'true', a metric will not be sent for this field.
//
// rate: one of 'gauge', 'counter', 'rate'
//	This tag dictates the metadata.RateType we send.
//
// unit: 'bytes', 'pages', etc
//	This tag dictates the metadata.Unit we send.
//
// metric:
//      This is the metric name which will be sent. If not present, the 'json'
//      tag is sent as the metric name.
//
// Special handling:
//
// Metrics having the json tag suffix of 'in_milliseconds' are automagically
// divided by 1000 and sent as seconds. The suffix is stripped from the name.
//
// Metrics having the json tag suffix of 'in_bytes' are automatically sent as
// gauge bytes. The suffix is stripped from the metric name.
func (s *structProcessor) add(prefix string, st interface{}, ts opentsdb.TagSet) {
	t := reflect.TypeOf(st)
	valueOf := reflect.ValueOf(st)
	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i)
		value := valueOf.Field(i).Interface()
		if field.Tag.Get("exclude") == "true" {
			continue
		}
		var (
			jsonTag    = field.Tag.Get("json")
			metricTag  = field.Tag.Get("metric")
			versionTag = field.Tag.Get("version")
			rateTag    = field.Tag.Get("rate")
			unitTag    = field.Tag.Get("unit")
		)
		metricName := jsonTag
		if metricTag != "" {
			metricName = metricTag
		}
		if metricName == "" {
			slog.Errorf("Unable to determine metric name for field %s. Skipping.", field.Name)
			continue
		}
		if versionTag == "" || strings.HasPrefix(s.elasticVersion, versionTag) {
			switch value := value.(type) {
			case int, float64: // Number types in our structs are only ints and float64s.
				// Turn all millisecond metrics into seconds
				if strings.HasSuffix(metricName, "_in_millis") {
					switch value.(type) {
					case int:
						value = float64(value.(int)) / 1000
					case float64:
						value = value.(float64) / 1000
					}
					unitTag = "seconds"
					metricName = strings.TrimSuffix(metricName, "_in_millis")
				}
				// Set rate and unit for all "_in_bytes" metrics, and strip the "_in_bytes"
				if strings.HasSuffix(metricName, "_in_bytes") {
					if rateTag == "" {
						rateTag = "gauge"
					}
					unitTag = "bytes"
					metricName = strings.TrimSuffix(metricName, "_in_bytes")
				}
				Add(s.md, prefix+"."+metricName, value, ts, metadata.RateType(rateTag), metadata.Unit(unitTag), field.Tag.Get("desc"))
			case string:
				// The json data has a lot of strings, and we don't care about em.
			default:
				// If we hit another struct, recurse
				if reflect.ValueOf(value).Kind() == reflect.Struct {
					s.add(prefix+"."+metricName, value, ts)
				} else {
					slog.Errorf("Field %s for metric %s is non-numeric type. Cannot record as a metric.\n", field.Name, prefix+"."+metricName)
				}
			}
		}
	}
}
コード例 #5
0
ファイル: service_windows.go プロジェクト: eswdd/bosun
func runService(name string, isDebug bool) {
	errFix := fixEventMessageFile(name) //Temp fix. Remove after a few weeks.
	if errFix != nil {
		slog.Errorf("%s fixEventMessageFile failed: %v", name, errFix)
		return
	}
	if isDebug {
		slog.SetEventLog(debug.New(name), 1)
	} else {
		elog, err := eventlog.Open(name)
		if err != nil {
			return
		}
		slog.SetEventLog(elog, 1)
		defer elog.Close()
	}
	slog.Infof("starting %s service version %v (%v)", name, version.Version, version.VersionSHA)
	run := svc.Run
	if isDebug {
		run = debug.Run
	}
	err := run(name, &s{})
	if err != nil {
		slog.Errorf("%s service failed: %v", name, err)
		return
	}
	slog.Infof("%s service stopped", name)
	os.Exit(0)
}
コード例 #6
0
ファイル: redis_counters.go プロジェクト: eswdd/bosun
func c_redis_counters(server string, db int) (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	conn, err := redis.Dial("tcp", server, redis.DialDatabase(db))
	if err != nil {
		return md, err
	}
	defer conn.Close()
	if _, err := conn.Do("CLIENT", "SETNAME", "scollector"); err != nil {
		return md, err
	}
	cursor := 0
	for {
		vals, err := redis.Values(conn.Do("HSCAN", collect.RedisCountersKey, cursor))
		if err != nil {
			return md, err
		}
		if len(vals) != 2 {
			return md, fmt.Errorf("Unexpected number of values")
		}
		cursor, err = redis.Int(vals[0], nil)
		if err != nil {
			return md, err
		}
		pairs, err := redis.StringMap(vals[1], nil)
		if err != nil {
			return md, err
		}
		for mts, val := range pairs {
			parts := strings.Split(mts, ":")
			if len(parts) != 2 {
				slog.Errorf("Invalid metric tag set counter: %s", mts)
				continue
			}
			metric := parts[0]
			tags, err := opentsdb.ParseTags(parts[1])
			if err != nil {
				slog.Errorf("Invalid tags: %s", parts[1])
				continue
			}
			v, err := strconv.Atoi(val)
			if err != nil {
				slog.Errorf("Invalid counter value: %s", val)
				continue
			}
			Add(&md, metric, v, tags, metadata.Counter, metadata.Count, "")
		}
		if cursor == 0 {
			break
		}
	}
	return md, nil
}
コード例 #7
0
ファイル: sched.go プロジェクト: harryshayne/bosun
func (s *Schedule) PutMetadata(k metadata.Metakey, v interface{}) {

	isCoreMeta := (k.Name == "desc" || k.Name == "unit" || k.Name == "rate")
	if !isCoreMeta {
		s.metaLock.Lock()
		s.Metadata[k] = &Metavalue{time.Now().UTC(), v}
		s.metaLock.Unlock()
		return
	}
	if k.Metric == "" {
		slog.Error("desc, rate, and unit require metric name")
		return
	}
	strVal, ok := v.(string)
	if !ok {
		slog.Errorf("desc, rate, and unit require value to be string. Found: %s", reflect.TypeOf(v))
		return
	}
	s.metricMetaLock.Lock()
	metricData, ok := s.metricMetadata[k.Metric]
	if !ok {
		metricData = &MetadataMetric{}
		s.metricMetadata[k.Metric] = metricData
	}
	switch k.Name {
	case "desc":
		metricData.Description = strVal
	case "unit":
		metricData.Unit = strVal
	case "rate":
		metricData.Type = strVal
	}
	s.metricMetaLock.Unlock()
}
コード例 #8
0
ファイル: funcs.go プロジェクト: jareksm/bosun
func timeTSDBRequest(e *State, T miniprofiler.Timer, req *opentsdb.Request) (s opentsdb.ResponseSet, err error) {
	e.tsdbQueries = append(e.tsdbQueries, *req)
	if e.autods > 0 {
		for _, q := range req.Queries {
			if q.Downsample == "" {
				if err := req.AutoDownsample(e.autods); err != nil {
					return nil, err
				}
			}
		}
	}
	b, _ := json.MarshalIndent(req, "", "  ")
	tries := 1
	for {
		T.StepCustomTiming("tsdb", "query", string(b), func() {
			getFn := func() (interface{}, error) {
				return e.tsdbContext.Query(req)
			}
			var val interface{}
			val, err = e.cache.Get(string(b), getFn)
			s = val.(opentsdb.ResponseSet).Copy()

		})
		if err == nil || tries == tsdbMaxTries {
			break
		}
		slog.Errorf("Error on tsdb query %d: %s", tries, err.Error())
		tries++
	}
	return
}
コード例 #9
0
ファイル: check.go プロジェクト: Skyscanner/bosun
func (s *Schedule) CheckAlert(T miniprofiler.Timer, r *RunHistory, a *conf.Alert) {
	slog.Infof("check alert %v start", a.Name)
	start := utcNow()
	for _, ak := range s.findUnknownAlerts(r.Start, a.Name) {
		r.Events[ak] = &models.Event{Status: models.StUnknown}
	}
	var warns, crits models.AlertKeys
	d, err := s.executeExpr(T, r, a, a.Depends)
	var deps expr.ResultSlice
	if err == nil {
		deps = filterDependencyResults(d)
		crits, err = s.CheckExpr(T, r, a, a.Crit, models.StCritical, nil)
		if err == nil {
			warns, err = s.CheckExpr(T, r, a, a.Warn, models.StWarning, crits)
		}
	}
	unevalCount, unknownCount := markDependenciesUnevaluated(r.Events, deps, a.Name)
	if err != nil {
		slog.Errorf("Error checking alert %s: %s", a.Name, err.Error())
		removeUnknownEvents(r.Events, a.Name)
		s.markAlertError(a.Name, err)
	} else {
		s.markAlertSuccessful(a.Name)
	}
	collect.Put("check.duration", opentsdb.TagSet{"name": a.Name}, time.Since(start).Seconds())
	slog.Infof("check alert %v done (%s): %v crits, %v warns, %v unevaluated, %v unknown", a.Name, time.Since(start), len(crits), len(warns), unevalCount, unknownCount)
}
コード例 #10
0
ファイル: elasticsearch.go プロジェクト: eswdd/bosun
func init() {
	registerInit(func(c *conf.Conf) {
		for _, filter := range c.ElasticIndexFilters {
			err := AddElasticIndexFilter(filter)
			if err != nil {
				slog.Errorf("Error processing ElasticIndexFilter: %s", err)
			}
		}
		collectors = append(collectors, &IntervalCollector{
			F: func() (opentsdb.MultiDataPoint, error) {
				return c_elasticsearch(false)
			},
			name:   "elasticsearch",
			Enable: enableURL("http://localhost:9200/"),
		})
		collectors = append(collectors, &IntervalCollector{
			F: func() (opentsdb.MultiDataPoint, error) {
				return c_elasticsearch(true)
			},
			name:     "elasticsearch-indices",
			Interval: time.Minute * 15,
			Enable:   enableURL("http://localhost:9200/"),
		})
	})
}
コード例 #11
0
ファイル: check.go プロジェクト: nicollet/bosun
func (s *Schedule) findUnknownAlerts(now time.Time, alert string) []models.AlertKey {
	keys := []models.AlertKey{}
	if utcNow().Sub(bosunStartupTime) < s.SystemConf.GetCheckFrequency() {
		return keys
	}
	if !s.AlertSuccessful(alert) {
		return keys
	}
	a := s.RuleConf.GetAlert(alert)
	t := a.Unknown
	if t == 0 {
		runEvery := s.SystemConf.GetDefaultRunEvery()
		if a.RunEvery != 0 {
			runEvery = a.RunEvery
		}
		t = s.SystemConf.GetCheckFrequency() * 2 * time.Duration(runEvery)
	}
	maxTouched := now.UTC().Unix() - int64(t.Seconds())
	untouched, err := s.DataAccess.State().GetUntouchedSince(alert, maxTouched)
	if err != nil {
		slog.Errorf("Error finding unknown alerts for alert %s: %s.", alert, err)
		return keys
	}
	for _, ak := range untouched {
		if a.Squelch.Squelched(ak.Group()) {
			continue
		}
		keys = append(keys, ak)
	}
	return keys
}
コード例 #12
0
ファイル: fastly.go プロジェクト: nicollet/bosun
func c_fastly(c fastlyClient) (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	to := time.Now().UTC().Truncate(time.Minute)
	from := to.Add(-15 * time.Minute) // "Minutely data will be delayed by roughly 10 to 15 minutes from the current time -- Fastly Docs"

	// Aggregate
	statsCollection, err := c.GetAggregateStats(from, to)
	if err != nil {
		return md, err
	}
	for _, stats := range statsCollection {
		fastlyReflectAdd(&md, "fastly", "", stats, stats.StartTime, nil)
	}

	// By Service
	services, err := c.GetServices()
	if err != nil {
		return md, err
	}
	for _, service := range services {
		statsCollection, err := c.GetServiceStats(from, to, service.Id)
		if err != nil {
			slog.Errorf("couldn't get stats for service %v with id %v: %v", service.Name, service.Id, err)
			continue
		}
		for _, stats := range statsCollection {
			fastlyReflectAdd(&md, "fastly", "_by_service", stats, stats.StartTime, service.TagSet())
		}
	}

	// By Region
	regions, err := c.GetRegions()
	if err != nil {
		return md, err
	}
	for _, region := range regions {
		statsCollection, err := c.GetRegionStats(from, to, region)
		if err != nil {
			slog.Errorf("couldn't get stats for region %v: %v", region, err)
			continue
		}
		for _, stats := range statsCollection {
			fastlyReflectAdd(&md, "fastly", "_by_region", stats, stats.StartTime, region.TagSet())
		}
	}
	return md, nil
}
コード例 #13
0
ファイル: eventListener.go プロジェクト: fcolmenero/bosun
func incrementRedisCounter(data []byte, addr string, conn redis.Conn) {
	if len(data) < 5 {
		slog.Errorf("Insufficient data for increment from %s.", addr)
		return
	}
	r := bytes.NewReader(data)
	var i int32
	err := binary.Read(r, binary.BigEndian, &i)
	if err != nil {
		slog.Error(err)
		return
	}
	mts := string(data[4:])
	if _, err = conn.Do("HINCRBY", RedisCountersKey, mts, i); err != nil {
		slog.Errorf("Error incrementing counter %s by %d. From %s. %s", mts, i, addr, err)
	}
}
コード例 #14
0
ファイル: check.go プロジェクト: Skyscanner/bosun
func (s *Schedule) GetUnknownAndUnevaluatedAlertKeys(alert string) (unknown, uneval []models.AlertKey) {
	unknown, uneval, err := s.DataAccess.State().GetUnknownAndUnevalAlertKeys(alert)
	if err != nil {
		slog.Errorf("Error getting unknown/unevaluated alert keys: %s", err)
		return nil, nil
	}
	return unknown, uneval
}
コード例 #15
0
ファイル: snmp.go プロジェクト: eswdd/bosun
// snmp_subtree takes an oid and returns all data exactly one level below it. It
// produces an error if there is more than one level below.
func snmp_subtree(host, community, oid string) (map[string]interface{}, error) {
	rows, err := snmp.Walk(host, community, oid)
	if err != nil {
		return nil, err
	}
	m := make(map[string]interface{})
	for rows.Next() {
		key := ""
		var a interface{}
		switch oid {
		case ifHCInBroadcastPkts:
			a = new(big.Int)
			id, err := rows.Scan(&a)
			if err != nil {
				slog.Errorf("Error scanning oid %v on host %v: %v", oid, host, err)
				continue
			}
			switch t := id.(type) {
			case int:
				key = fmt.Sprint(t)
			default:
				return nil, fmt.Errorf("snmp subtree: only one level allowed")
			}
		default:
			id, err := rows.Scan(&a)
			if err != nil {
				slog.Errorf("Error scanning oid %v on host %v: %v", oid, host, err)
				continue
			}
			switch t := id.(type) {
			case int:
				key = fmt.Sprint(t)
			case []int:
				key = snmpOidArrayToString(t)
			default:
				return nil, fmt.Errorf("Unknown key type: %s", reflect.TypeOf(id).String())
			}
		}
		m[key] = a
	}
	if err := rows.Err(); err != nil && err != io.EOF {
		return nil, err
	}
	return m, nil
}
コード例 #16
0
ファイル: fastly.go プロジェクト: nicollet/bosun
func fastlyReflectAdd(md *opentsdb.MultiDataPoint, prefix, suffix string, st interface{}, timeStamp int64, ts opentsdb.TagSet) {
	t := reflect.TypeOf(st)
	valueOf := reflect.ValueOf(st)
	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i)
		value := valueOf.Field(i).Interface()
		var (
			jsonTag   = field.Tag.Get("json")
			metricTag = field.Tag.Get("metric")
			rateTag   = field.Tag.Get("rate")
			unitTag   = field.Tag.Get("unit")
			divTag    = field.Tag.Get("div")
			descTag   = field.Tag.Get("desc")
			exclude   = field.Tag.Get("exclude") != ""
		)
		if exclude || descTag == "" {
			continue
		}
		metricName := jsonTag
		if metricTag != "" {
			metricName = metricTag
		}
		if metricName == "" {
			slog.Errorf("Unable to determine metric name for field %s. Skipping.", field.Name)
			continue
		}
		shouldDiv := divTag != ""
		if shouldDiv {
			descTag = fmt.Sprintf("%v %v", descTag, fastlyDivDesc)
		}
		fullMetric := fmt.Sprintf("%v.%v%v", prefix, metricName, suffix)
		switch value := value.(type) {
		case int64, float64:
			var v float64
			if f, found := value.(float64); found {
				v = f
			} else {
				v = float64(value.(int64))
			}
			if shouldDiv {
				v /= 60.0
			}
			AddTS(md, fullMetric, timeStamp, v, ts, metadata.RateType(rateTag), metadata.Unit(unitTag), descTag)
		case string:
			// Floats in strings, I know not why, precision perhaps?
			// err ignored since we expect non number strings in the struct
			if f, err := strconv.ParseFloat(value, 64); err != nil {
				if shouldDiv {
					f /= 60.0
				}
				AddTS(md, fullMetric, timeStamp, f, ts, metadata.RateType(rateTag), metadata.Unit(unitTag), descTag)
			}
		default:
			// Pass since there is no need to recurse
		}
	}
}
コード例 #17
0
ファイル: snmp_cisco.go プロジェクト: eswdd/bosun
func c_cisco_nxos(host, community string, cpuIntegrator tsIntegrator) (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	ts := opentsdb.TagSet{"host": host}
	// CPU
	if err := ciscoCPU(host, community, ts, cpuIntegrator, &md); err != nil {
		return md, err
	}
	// Memory
	memRaw, err := snmp_subtree(host, community, ciscoBaseOID+cpmCPUTotalEntry)
	if err != nil {
		return md, fmt.Errorf("failed to get cpmCPUTotalEntry (for memory) for host %v: %v", host, err)
	}
	var usedMem, freeMem, totalMem int64
	var usedOk, freeOk bool
	for id, value := range memRaw {
		var v int64
		switch id {
		case "12.1":
			if v, usedOk = value.(int64); usedOk {
				usedMem = v * 2 << 9 // KiB to Bytes
				totalMem += usedMem
				Add(&md, osMemUsed, usedMem, ts, metadata.Gauge, metadata.Bytes, osMemUsedDesc)
			} else {
				slog.Errorf("failed to convert used memory %v to int64 for host %v", value, host)
			}
		case "13.1":
			if v, freeOk = value.(int64); freeOk {
				freeMem = v * 2 << 9
				totalMem += freeMem
				Add(&md, osMemFree, freeMem, ts, metadata.Gauge, metadata.Bytes, osMemFreeDesc)
			} else {
				slog.Errorf("failed to convert free memory %v to int64 for host %v", value, host)
			}
		}
	}
	if usedOk && freeOk {
		Add(&md, osMemTotal, totalMem, ts, metadata.Gauge, metadata.Bytes, osMemTotalDesc)
		Add(&md, osMemPctFree, int64(float64(freeMem)/float64(totalMem)*100), ts, metadata.Gauge, metadata.Pct, osMemPctFreeDesc)
	} else {
		slog.Errorf("failed to get both free and used memory for host %v", host)
	}
	return md, nil
}
コード例 #18
0
ファイル: check.go プロジェクト: nicollet/bosun
func (s *Schedule) CheckAlert(T miniprofiler.Timer, r *RunHistory, a *conf.Alert) (cancelled bool) {
	slog.Infof("check alert %v start", a.Name)
	start := utcNow()
	for _, ak := range s.findUnknownAlerts(r.Start, a.Name) {
		r.Events[ak] = &models.Event{Status: models.StUnknown}
	}
	var warns, crits models.AlertKeys
	type res struct {
		results *expr.Results
		error   error
	}
	// buffered channel so go func that runs executeExpr won't leak if the Check is cancelled
	// by the closing of the schedule
	rc := make(chan res, 1)
	var d *expr.Results
	var err error
	go func() {
		d, err := s.executeExpr(T, r, a, a.Depends)
		rc <- res{d, err} // this will hang forever if the channel isn't buffered since nothing will ever receieve from rc
	}()
	select {
	case res := <-rc:
		d = res.results
		err = res.error
	// If the schedule closes before the expression has finised executing, we abandon the
	// execution of the expression
	case <-s.runnerContext.Done():
		return true
	}
	var deps expr.ResultSlice
	if err == nil {
		deps = filterDependencyResults(d)
		crits, err, cancelled = s.CheckExpr(T, r, a, a.Crit, models.StCritical, nil)
		if err == nil && !cancelled {
			warns, err, cancelled = s.CheckExpr(T, r, a, a.Warn, models.StWarning, crits)
		}
	}
	if cancelled {
		return true
	}
	unevalCount, unknownCount := markDependenciesUnevaluated(r.Events, deps, a.Name)
	if err != nil {
		slog.Errorf("Error checking alert %s: %s", a.Name, err.Error())
		removeUnknownEvents(r.Events, a.Name)
		s.markAlertError(a.Name, err)
	} else {
		s.markAlertSuccessful(a.Name)
	}
	collect.Put("check.duration", opentsdb.TagSet{"name": a.Name}, time.Since(start).Seconds())
	slog.Infof("check alert %v done (%s): %v crits, %v warns, %v unevaluated, %v unknown", a.Name, time.Since(start), len(crits), len(warns), unevalCount, unknownCount)
	return false
}
コード例 #19
0
ファイル: collectors.go プロジェクト: nicollet/bosun
// AddTS is the same as Add but lets you specify the timestamp
func AddTS(md *opentsdb.MultiDataPoint, name string, ts int64, value interface{}, t opentsdb.TagSet, rate metadata.RateType, unit metadata.Unit, desc string) {
	// Check if we really want that metric
	if skipMetric(name) {
		return
	}

	tags := t.Copy()
	if host, present := tags["host"]; !present {
		tags["host"] = util.Hostname
	} else if host == "" {
		delete(tags, "host")
	}
	// if tags are not cleanable, log a message and skip it
	if err := tags.Clean(); err != nil {
		line := ""
		//attempt to log where Add was called from
		if _, filename, l, ok := runtime.Caller(1); ok {
			if filepath.Base(filename) == "collectors.go" {
				_, filename, l, ok = runtime.Caller(2)
			}
			if ok {
				line = fmt.Sprintf("%s:%d", filepath.Base(filename), l)
			}
		}
		slog.Errorf("Invalid tagset discovered: %s. Skipping datapoint. Added from: %s", tags.String(), line)
		return
	}
	if rate != metadata.Unknown {
		metadata.AddMeta(name, nil, "rate", rate, false)
	}
	if unit != metadata.None {
		metadata.AddMeta(name, nil, "unit", unit, false)
	}
	if desc != "" {
		metadata.AddMeta(name, tags, "desc", desc, false)
	}
	tags = AddTags.Copy().Merge(tags)
	if b, ok := value.(bool); ok {
		if b {
			value = 1
		} else {
			value = 0
		}
	}
	d := opentsdb.DataPoint{
		Metric:    name,
		Timestamp: ts,
		Value:     value,
		Tags:      tags,
	}
	*md = append(*md, &d)
}
コード例 #20
0
ファイル: disk_linux.go プロジェクト: eswdd/bosun
func isPseudoFS(name string) (res bool) {
	err := readLine("/proc/filesystems", func(s string) error {
		ss := strings.Split(s, "\t")
		if len(ss) == 2 && ss[1] == name && ss[0] == "nodev" {
			res = true
		}
		return nil
	})
	if err != nil {
		slog.Errorf("can not read '/proc/filesystems': %v", err)
	}
	return
}
コード例 #21
0
ファイル: disk_linux.go プロジェクト: rajder/bosun
func isPseudoFS(name string) (res bool) {
	err := readLine("/proc/filesystems", func(s string) error {
		if strings.Contains(s, name) && strings.Contains(s, "nodev") {
			res = true
			return nil
		}
		return nil
	})
	if err != nil {
		slog.Errorf("can not read '/proc/filesystems': %v", err)
	}
	return
}
コード例 #22
0
ファイル: redis_counters.go プロジェクト: nicollet/bosun
func c_redis_counters(server string, db int) (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	conn, err := redis.Dial("tcp", server, redis.DialDatabase(db))
	if err != nil {
		return md, slog.Wrap(err)
	}
	defer conn.Close()

	//do a dance to detect proper hscan command for ledis or redis
	hscanCmd := "XHSCAN"
	info, err := redis.String(conn.Do("info", "server"))
	if err != nil {
		return md, slog.Wrap(err)
	}
	if strings.Contains(info, "redis_version") {
		hscanCmd = "HSCAN"
	}

	cursor := "0"
	for {
		vals, err := redis.Values(conn.Do(hscanCmd, collect.RedisCountersKey, cursor))
		if err != nil {
			return md, slog.Wrap(err)
		}
		if len(vals) != 2 {
			return md, fmt.Errorf("Unexpected number of values")
		}
		cursor, err = redis.String(vals[0], nil)
		if err != nil {
			return md, slog.Wrap(err)
		}
		pairs, err := redis.StringMap(vals[1], nil)
		if err != nil {
			return md, slog.Wrap(err)
		}
		for key, val := range pairs {
			ak := models.AlertKey(key)

			v, err := strconv.Atoi(val)
			if err != nil {
				slog.Errorf("Invalid counter value: %s", val)
				continue
			}
			Add(&md, ak.Name(), v, ak.Group(), metadata.Counter, metadata.Count, "")
		}
		if cursor == "" || cursor == "0" {
			break
		}
	}
	return md, nil
}
コード例 #23
0
ファイル: snmp_ips.go プロジェクト: eswdd/bosun
func c_snmp_ips(community, host string) (opentsdb.MultiDataPoint, error) {
	ifIPAdEntAddrRaw, err := snmp_subtree(host, community, ifIPAdEntAddr)
	if err != nil {
		return nil, err
	}
	ipAdEnts := make(map[string]*ipAdEntAddr)
	for id, value := range ifIPAdEntAddrRaw {
		// Split entry type id from ip address
		sp := strings.SplitN(id, ".", 2)
		if len(sp) != 2 {
			slog.Errorln("unexpected length of snmp resonse")
		}
		typeId := sp[0]
		address := sp[1]
		if _, ok := ipAdEnts[address]; !ok {
			ipAdEnts[address] = &ipAdEntAddr{}
		}
		switch typeId {
		case "1":
			if v, ok := value.([]byte); ok {
				ipAdEnts[address].IP = v
			}
		case "2":
			if v, ok := value.(int64); ok {
				ipAdEnts[address].InterfaceId = v
			}
		case "3":
			if v, ok := value.([]byte); ok {
				ipAdEnts[address].Mask = v
			}
		}
	}
	ipsByInt := make(map[int64][]net.IPNet)
	for _, ipNet := range ipAdEnts {
		ipsByInt[ipNet.InterfaceId] = append(ipsByInt[ipNet.InterfaceId], ipNet.IPNet)
	}
	for intId, ipNets := range ipsByInt {
		var ips []string
		for _, ipNet := range ipNets {
			ips = append(ips, ipNet.String())
		}
		sort.Strings(ips)
		j, err := json.Marshal(ips)
		if err != nil {
			slog.Errorf("error marshaling ips for host %v: %v", host, err)
		}
		metadata.AddMeta("", opentsdb.TagSet{"host": host, "iface": fmt.Sprintf("%v", intId)}, "addresses", string(j), false)
	}
	return nil, nil
}
コード例 #24
0
ファイル: eventListener.go プロジェクト: fcolmenero/bosun
/*
 Listen on the specified udp port for events.
 This provides long term aggrigation for sparse events.
 wire format: opcode(1 byte) | data

Opcodes:
 1: increment - increments a redis counter for the specified metric/tag set
     data: count(4 bytes signed int) | metric:tag1=foo,tag2=bar
*/
func ListenUdp(port int, redisHost string, redisDb int) error {
	addr := net.UDPAddr{
		Port: port,
		IP:   net.ParseIP("127.0.0.1"),
	}
	conn, err := net.ListenUDP("udp", &addr)
	if err != nil {
		return err
	}
	pool := newRedisPool(redisHost, redisDb)
	for {
		buf := make([]byte, 1025)
		n, addr, err := conn.ReadFromUDP(buf)
		if err != nil {
			slog.Error(err)
			continue
		}
		if n == len(buf) { // if we get a full buffer, assume some was truncated.
			slog.Errorf("Too large a udp packet received from: %s. Skipping.", addr.String())
			continue
		}
		Add("udp.packets", opentsdb.TagSet{}, 1)
		go func(data []byte, addr string) {
			c := pool.Get()
			defer c.Close()
			if len(data) == 0 {
				slog.Errorf("Empty packet received from %s.", addr)
			}
			switch data[0] {
			case 1:
				incrementRedisCounter(data[1:], addr, c)
			default:
				slog.Errorf("Unknown opcode %d from %s.", data[0], addr)
			}
		}(buf[:n], addr.String())
	}
}
コード例 #25
0
ファイル: database.go プロジェクト: MichaelS11/bosun
func DatabaseCollect(c conf.Database, collectorStatsChan chan<- *ContinuousCollectorStats) {
	queries := DatabaseGetQueries(c)

	if len(queries) < 1 {
		slog.Errorf("%v: %v: %v", "Database", c.Type, "no queries")
	}

	db, err := DatabaseConnect(c)
	if err != nil {
		return
	}

	defer db.Close()

	var waitGroup sync.WaitGroup
	queryChan := make(chan *DatabaseQuery, 1)

	waitGroup.Add(c.MaxOpenConns)
	for i := 0; i < c.MaxOpenConns; i++ {
		go func() {
			DatabaseRunQueryWorker(c, db, queryChan, collectorStatsChan)
			waitGroup.Done()
		}()
	}

Loop:
	for {
		nextTime := queries[0].next
		// maybe change to a priority queue instead of checking each slice?
		for i := range queries {
			if time.Now().After(queries[i].next) {
				queryChan <- &queries[i]
				queries[i].next = time.Now().Add(queries[i].interval)
				if queries[i].next.Before(nextTime) {
					nextTime = queries[i].next
				}
			}
		}
		next := time.After(nextTime.Sub(time.Now()))
		select {
		case <-next:
		case <-ContinuousCollectorVars.quit:
			break Loop
		}
	}

	close(queryChan)
	waitGroup.Wait()
}
コード例 #26
0
ファイル: cadvisor.go プロジェクト: mhenderson-so/bosun
func c_cadvisor(c *client.Client, config *conf.Cadvisor) (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint

	containers, err := c.AllDockerContainers(&v1.ContainerInfoRequest{NumStats: 1})
	if err != nil {
		slog.Errorf("Error fetching containers from cadvisor: %v", err)
		return md, err
	}

	for _, container := range containers {
		statsForContainer(&md, &container, config)
	}

	return md, nil
}
コード例 #27
0
ファイル: interval.go プロジェクト: noblehng/bosun
func (c *IntervalCollector) Run(dpchan chan<- *opentsdb.DataPoint, quit <-chan struct{}) {
	if c.Enable != nil {
		go func() {
			for {
				next := time.After(time.Minute * 5)
				c.Lock()
				c.enabled = c.Enable()
				c.Unlock()
				<-next
			}
		}()
	}

	for {
		interval := c.Interval
		if interval == 0 {
			interval = DefaultFreq
		}
		next := time.After(interval)
		if c.Enabled() {
			timeStart := time.Now()
			md, err := c.F()
			timeFinish := time.Since(timeStart)
			result := 0
			if err != nil {
				slog.Errorf("%v: %v", c.Name(), err)
				result = 1
			}
			if !collect.DisableDefaultCollectors {
				tags := opentsdb.TagSet{"collector": c.Name(), "os": runtime.GOOS}
				Add(&md, "scollector.collector.duration", timeFinish.Seconds(), tags, metadata.Gauge, metadata.Second, "Duration in seconds for each collector run.")
				Add(&md, "scollector.collector.error", result, tags, metadata.Gauge, metadata.Ok, "Status of collector run. 1=Error, 0=Success.")
			}
			for _, dp := range md {
				c.ApplyTagOverrides(dp.Tags)
				dpchan <- dp
			}
		}
		select {
		case <-next:
		case <-quit:
			return
		}

	}
}
コード例 #28
0
ファイル: check.go プロジェクト: Skyscanner/bosun
// RunHistory processes an event history and triggers notifications if needed.
func (s *Schedule) RunHistory(r *RunHistory) {
	checkNotify := false
	silenced := s.Silenced()
	for ak, event := range r.Events {
		shouldNotify, err := s.runHistory(r, ak, event, silenced)
		checkNotify = checkNotify || shouldNotify
		if err != nil {
			slog.Errorf("Error in runHistory for %s. %s.", ak, err)
		}
	}
	if checkNotify && s.nc != nil {
		select {
		case s.nc <- true:
		default:
		}
	}
}
コード例 #29
0
ファイル: disk_linux.go プロジェクト: nicollet/bosun
func checkMdadmLinux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint

	volumes, err := filepath.Glob("/dev/md*")
	if err != nil {
		return md, err
	}
	for _, volume := range filterVolumes(volumes) {
		detail, err := examineMdadmVolume(volume)
		if err != nil {
			slog.Errorf("mdadm: can't parse %s data, %s", volume, err)
			continue
		}
		addMdadmMetric(&md, volume, detail)
	}
	return md, nil
}
コード例 #30
0
ファイル: service_windows.go プロジェクト: eswdd/bosun
func (m *s) Execute(args []string, r <-chan svc.ChangeRequest, changes chan<- svc.Status) (ssec bool, errno uint32) {
	const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown
	changes <- svc.Status{State: svc.StartPending}
	changes <- svc.Status{State: svc.Running, Accepts: cmdsAccepted}
loop:
	for c := range r {
		switch c.Cmd {
		case svc.Interrogate:
			changes <- c.CurrentStatus
		case svc.Stop, svc.Shutdown:
			break loop
		default:
			slog.Errorf("unexpected control request #%d", c)
		}
	}
	changes <- svc.Status{State: svc.StopPending}
	return
}