//Sets any invalid fields to their default value
func PerformDefault(conf Configuration) {
err := conf.Validate()
name := reflect.ValueOf(conf).Type()
if err != nil {
if errs, ok := err.(validator.ErrorMap); ok {
for fieldName := range errs {
if strings.Contains(fieldName, ".") {
//If field Name contains a '.' than it is a subfield of a current field
continue
}
glog.V(4).Info("Invalid field searching for default ", fieldName)
if ok, err := defaults.HasDefault(conf, fieldName); ok {
value, err := defaults.SetDefault(conf, fieldName)
if err != nil {
glog.Errorf("Failed to set default of %s on %s", fieldName, name)
} else {
glog.Infof("Defaulted %v.%s to '%v'", name, fieldName, value)
}
} else {
glog.V(4).Info("No default found: ", err)
}
}
} else {
glog.Errorf("Failed to validate %s: %s", name, err)
}
}
}
func (self *ScheduledQuery) callback(start, stop time.Time) {
query := self.builder.GetForTimeRange(start, stop)
if self.callback != nil {
glog.V(1).Info("Scheduling query:", query)
self.Callback(query)
}
}
// Determine if the window is anomalous
func (self *Detector) IsAnomalous(window *Window) bool {
self.Stats.WindowCount++
self.Stats.DataPointCount += uint64(len(window.Data))
vote := 0.0
for _, fc := range self.counters {
fingerprint := fc.fingerprinter.Fingerprint(window.Copy())
count := fc.counter.Count(fingerprint)
if count < self.normalCount {
vote++
}
}
vote /= float64(len(self.counters))
anomalous := vote >= self.consensus
glog.V(3).Infof("Window anomalous: %v vote: %f, consensus: %f", anomalous, vote, self.consensus)
if anomalous {
self.Stats.AnomalousCount++
}
return anomalous
}
func (self *Schedule) Start() error {
if self.running {
return errors.New("Schedule already started")
}
self.running = true
stop := time.Now()
if self.Period > day {
stop = stop.Truncate(day)
} else {
stop = stop.Truncate(self.Period)
}
stop = stop.Add(self.Period)
go func(stop time.Time, period time.Duration) {
now := time.Now()
glog.V(2).Info("Starting schedule", stop.Add(self.Delay), stop.Add(self.Delay).Sub(now))
time.Sleep(stop.Add(self.Delay).Sub(now))
ticker := time.NewTicker(period)
for self.running {
self.Callback(stop.Add(-period), stop)
stop = stop.Add(period)
<-ticker.C
}
ticker.Stop()
}(stop, self.Period)
return nil
}
// Register a Factory by name
func (self *Registery) RegisterFactory(name string, factory Factory) error {
glog.V(2).Infof("Registering Factory %s", name)
if _, ok := self.factories[name]; ok {
return errors.New(fmt.Sprintf("Factory of name %s already registered"))
}
self.factories[name] = factory
return nil
}
func (self *Sigma) Fingerprint(window morgoth.Window) morgoth.Fingerprint {
mean, std := calcStats(window.Data)
glog.V(4).Infof("N: %d Mean: %f Std: %f", len(window.Data), mean, std)
return SigmaFingerprint{
mean: mean,
threshold: self.deviations * std,
}
}
func (self *Manager) processQueries() {
for query := range self.queryQueue {
glog.V(2).Info("Executing query:", query)
windows, err := self.engine.GetWindows(query.query)
if err != nil {
glog.Errorf("Failed to execute query: '%s' %s", query, err)
continue
}
// Tag windows with query tags
glog.V(3).Info("Adding query tags: ", query.tags)
for _, w := range windows {
for t, v := range query.tags {
w.Tags[t] = v
}
}
self.ProcessWindows(windows)
}
}
func (self *AlertsManager) processQueries() {
for query := range self.queryQueue {
glog.V(2).Info("Executing query:", query)
windows, err := self.engine.GetWindows(query.query)
if err != nil {
glog.Errorf("Failed to execute query: '%s' %s", query, err)
continue
}
glog.Info(windows)
for _, w := range windows {
sum := 0.0
for _, point := range w.Data {
sum += point
}
glog.V(1).Info("Alert Query total ", sum)
if sum > query.threshold {
for _, n := range query.notifiers {
n.Notify(query.message, w)
}
}
}
}
}
func (self Config) Validate() (err error) {
glog.V(2).Info("Validating Config")
err = validator.Validate(self)
if err != nil {
return
}
err = self.EngineConf.Validate()
if err != nil {
return
}
for _, mp := range self.Mappings {
err = mp.Validate()
if err != nil {
return
}
}
return
}
func (self *APIServer) Start() (err error) {
self.handler = rest.ResourceHandler{
EnableStatusService: true,
}
self.handler.SetRoutes(
&rest.Route{"GET", "/status", self.status},
&rest.Route{"GET", "/stats", self.stats},
)
self.listener, err = net.Listen("tcp", fmt.Sprintf(":%d", self.port))
if err != nil {
glog.Errorf("Error binding APIServer: %s", err)
return
}
go func() {
err = http.Serve(self.listener, &self.handler)
if err != nil {
glog.Errorf("Error while APIServer was running: %s", err)
}
}()
glog.V(1).Info("APIServer running")
return
}
func (self *Manager) Start() (err error) {
for _, sq := range self.scheduledDataQueries {
sq.Start(self.queryQueue)
}
self.db, err = bolt.Open("morgoth.db", 0600, nil)
if err != nil {
return
}
self.db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte("morgoth"))
if err != nil {
return err
}
return nil
})
self.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("morgoth"))
mappingBytes := bytes.NewBuffer(b.Get([]byte("mappings")))
dec := gob.NewDecoder(mappingBytes)
mappings := make([]*DetectorMapper, 0)
err := dec.Decode(&mappings)
if err != nil {
return err
}
for _, m := range mappings {
self.mapper.addDetectorMapper(m)
}
return nil
})
glog.V(1).Infof("Starting %d processQueries routines", workerCount)
for i := 0; i < workerCount; i++ {
go self.processQueries()
}
return
}
func (self *App) Run() (err error) {
glog.Info("Starting app")
glog.Info("Setup signal handler")
go self.signalHandler()
glog.Info("Setup Engine")
self.engine, err = self.config.EngineConf.GetEngine()
if err != nil {
return
}
err = self.engine.Initialize()
if err != nil {
return
}
glog.Info("Setup Mapper")
detectorMatchers := make([]*DetectorMatcher, len(self.config.Mappings))
for i, mapping := range self.config.Mappings {
namePattern, err := regexp.Compile(mapping.Name)
if err != nil {
return fmt.Errorf("Error parsing regexp: %s: %s", mapping.Name, err)
}
tagPatterns := make(map[string]*regexp.Regexp, len(mapping.Tags))
for tag, pattern := range mapping.Tags {
r, err := regexp.Compile(pattern)
if err != nil {
return fmt.Errorf("Error parsing regexp: %s: %s", pattern, err)
}
tagPatterns[tag] = r
}
fingerprinters := make([]Fingerprinter, len(mapping.Detector.Fingerprinters))
for i, fp := range mapping.Detector.Fingerprinters {
f, err := fp.GetFingerprinter()
if err != nil {
return fmt.Errorf("Error creating Fingerprinter: %s", err)
}
fingerprinters[i] = f
}
detectorBuilder := NewDetectorBuilder(
mapping.Detector.NormalCount,
mapping.Detector.Consensus,
mapping.Detector.MinSupport,
mapping.Detector.ErrorTolerance,
fingerprinters,
)
glog.V(1).Infof("Created detector builder: %v", mapping.Detector)
matcher := NewDetectorMatcher(namePattern, tagPatterns, detectorBuilder)
glog.V(1).Infof("Created detector matcher: %v", matcher)
detectorMatchers[i] = matcher
}
detectorMappers := make([]*DetectorMapper, 0)
self.mapper = NewMapper(detectorMappers, detectorMatchers)
self.Stats.MapperStats = &self.mapper.Stats
glog.Info("Setup Manager")
scheduledDataQueries := make([]*scheduledDataQuery, len(self.config.Schedules))
for i, sc := range self.config.Schedules {
builder, err := self.engine.NewQueryBuilder(sc.Query, sc.GroupByInterval)
if err != nil {
return fmt.Errorf("Invalid query string: '%s', %s", sc.Query, err)
}
q := &scheduledDataQuery{
sq: NewScheduledQuery(
builder,
sc.Delay,
sc.Period,
),
tags: sc.Tags,
}
scheduledDataQueries[i] = q
}
self.manager = NewManager(self.mapper, self.engine, scheduledDataQueries)
glog.Info("Starting Manager...")
self.manager.Start()
glog.Infof("Starting APIServer on :%d", self.config.APIPort)
self.apiServer = NewAPIServer(self, self.config.APIPort)
self.apiServer.Start()
glog.Infof("Starting Alert Manger...")
scheduledAlertQueries := make([]*scheduledAlertQuery, len(self.config.Alerts))
for i, ac := range self.config.Alerts {
builder, err := self.engine.NewQueryBuilder(ac.Query, ac.GroupByInterval)
if err != nil {
return fmt.Errorf("Invalid query string: '%s', %s", ac.Query, err)
}
notifiers := make([]Notifier, len(ac.Notifiers))
for i, nc := range ac.Notifiers {
n, err := nc.GetNotifier()
if err != nil {
return fmt.Errorf("Invalid Notifier for query: '%s' Err: %s", ac.Query, err)
}
notifiers[i] = n
}
q := &scheduledAlertQuery{
sq: NewScheduledQuery(
builder,
ac.Delay,
ac.Period,
),
threshold: ac.Threshold,
notifiers: notifiers,
message: ac.Message,
}
scheduledAlertQueries[i] = q
}
self.alertsManager = NewAlertsManager(self.engine, scheduledAlertQueries)
self.alertsManager.Start()
// Wait for shutdown
<-self.shutdownChan
// Begin shutdown
self.manager.Stop()
self.apiServer.Stop()
glog.Info("App shutdown")
return
}
func (self *InfluxDBEngine) GetWindows(query morgoth.Query) ([]*morgoth.Window, error) {
con, err := client.NewClient(self.conf)
if err != nil {
return nil, err
}
q := client.Query{
Command: query.Command,
Database: self.database,
}
glog.V(3).Infof("Q: %s", query)
response, err := con.Query(q)
if err != nil {
return nil, err
}
if response.Err != nil {
return nil, response.Err
}
windowCount := 0
for _, result := range response.Results {
if result.Err != nil {
return nil, result.Err
}
windowCount += len(result.Series)
}
windows := make([]*morgoth.Window, windowCount)
glog.V(3).Infof("Results: %v", response.Results)
i := 0
for _, result := range response.Results {
for _, row := range result.Series {
w := &morgoth.Window{
Name: row.Name,
Data: make([]float64, len(row.Values)),
Tags: row.Tags,
Start: query.Start,
Stop: query.Stop,
}
//Find non time column
if len(row.Columns) != 2 {
return nil, errors.New("Queries must select only two columns, a time column and a numeric column")
}
numberColumn := 0
for c, name := range row.Columns {
if name != "time" {
numberColumn = c
break
}
}
for j, point := range row.Values {
//We care only about the value not the time
//TODO check columns
value := point[numberColumn]
if value != nil {
p := value.(json.Number)
v, err := p.Float64()
if err != nil {
return nil, err
}
w.Data[j] = v
}
}
windows[i] = w
i++
glog.V(4).Infof("W: %v", w)
}
}
glog.V(3).Infof("Windows: %v", windows)
return windows, nil
}