Esempio n. 1
0
func detectInitd(root string) (InitSystem, error) {
	ctlpath := root + "etc/init.d/"
	result, err := util.FileExists(ctlpath)
	if err != nil {
		return nil, err
	}

	if !result {
		util.Debug("init.d not detected in " + ctlpath)
		return nil, nil
	}

	matches, err := filepath.Glob(ctlpath + "*")
	if err != nil {
		return nil, err
	}

	if !result {
		util.Debug("init.d not detected in " + ctlpath)
		return nil, nil
	}

	if len(matches) > 0 {
		util.Info("Detected init.d in " + ctlpath)
		return &Initd{ctlpath, root + "var/run/", pidForString}, nil
	}

	util.Info(ctlpath + " exists but appears to be empty")
	return nil, nil
}
Esempio n. 2
0
/*
 * So many hacks in this.  OSX support can be seen as "bad" at best.
 */
func (ps *processStorage) capturePs(pid int) error {
	cmd := exec.Command("ps", "So", "rss,time,utime", "-p", strconv.Itoa(pid))
	sout, err := util.SafeRun(cmd)
	if err != nil {
		return err
	}

	lines, err := util.ReadLines(sout)
	if err != nil {
		return err
	}

	if len(lines) < 2 {
		return errors.New("Insufficient output from ps")
	}

	fields := strings.Fields(lines[1])
	val, err := strconv.ParseInt(fields[0], 10, 64)
	if err != nil {
		return err
	}

	ps.Save("memory", "rss", float64(1024*val))

	times := timeRegexp.FindStringSubmatch(fields[1])
	if times == nil {
		util.Debug("Unable to parse CPU time in " + lines[1])
		return nil
	}
	min, _ := strconv.ParseUint(times[1], 10, 32)
	sec, _ := strconv.ParseUint(times[2], 10, 32)
	cs, _ := strconv.ParseUint(times[3], 10, 32)

	ticks := min*60*100 + sec*100 + cs

	times = timeRegexp.FindStringSubmatch(fields[2])
	if times == nil {
		util.Debug("Unable to parse User time in " + lines[1])
		return nil
	}
	min, _ = strconv.ParseUint(times[1], 10, 32)
	sec, _ = strconv.ParseUint(times[2], 10, 32)
	cs, _ = strconv.ParseUint(times[3], 10, 32)

	uticks := min*60*100 + sec*100 + cs

	ps.Save("cpu", "user", float64(uticks))
	ps.Save("cpu", "system", float64(ticks-uticks))

	return nil
}
Esempio n. 3
0
func (hs *hostStorage) collectDisk(path string) error {
	var lines []string

	if path == "" {
		cmd := exec.Command("df", "-P")
		sout, err := util.SafeRun(cmd)
		if err != nil {
			return err
		}
		lines, err = util.ReadLines(sout)
		if err != nil {
			return err
		}
	} else {
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		lines, err = util.ReadLines(data)
		if err != nil {
			return err
		}
	}

	usage := map[string]float64{}

	for _, line := range lines {
		if line[0] == '/' {
			items := strings.Fields(line)
			if len(items) < 5 {
				util.Debug("Cannot parse df output: %v", items)
				continue
			}
			pct := items[4]
			if pct[len(pct)-1] == '%' {
				val, err := strconv.ParseInt(pct[0:len(pct)-1], 10, 32)
				if err != nil {
					util.Debug("Cannot parse df output: " + line)
				}
				usage[items[len(items)-1]] = float64(val)
			}

		}
	}

	for name, used := range usage {
		hs.saveType("disk", name, used, Gauge)
	}
	return nil
}
Esempio n. 4
0
func (l *Launchd) LookupService(serviceName string) (*ProcessStatus, error) {
	cmd := exec.Command("launchctl", "list")
	sout, err := util.SafeRun(cmd)
	if err != nil {
		return nil, &ServiceError{l.Name(), serviceName, err}
	}

	lines, err := util.ReadLines(sout)
	if err != nil {
		return nil, &ServiceError{l.Name(), serviceName, err}
	}

	for _, line := range lines {
		if strings.Contains(line, serviceName) {
			util.Debug("launchctl found " + serviceName)
			parts := strings.SplitN(line, "\t", 3)
			pid, err := strconv.ParseInt(parts[0], 10, 32)
			if err != nil {
				return nil, &ServiceError{l.Name(), serviceName, err}
			}

			return &ProcessStatus{int(pid), Up}, nil
		}
	}

	path := l.resolvePlist(serviceName)
	if path != "" {
		return &ProcessStatus{0, Down}, nil
	}

	return nil, &ServiceError{l.Name(), serviceName, ErrServiceNotFound}
}
Esempio n. 5
0
/*
  Resolve each defined service to its managing init system.  Called only
  at startup, this is what maps services to init and fires ProcessDoesNotExist events.
*/
func (svc *Service) Resolve(mgrs []services.InitSystem) error {
	for _, sm := range mgrs {
		// TODO There's a bizarre race condition here. Figure out
		// why this is necessary.  We shouldn't be multi-threaded yet.
		if sm == nil {
			continue
		}

		ps, err := sm.LookupService(svc.Name())
		if err != nil {
			serr := err.(*services.ServiceError)
			if serr.Err == services.ErrServiceNotFound {
				util.Debug(sm.Name() + " doesn't have " + svc.Name())
				continue
			}
			return err
		}
		util.Info("Found %s/%s with status %s", sm.Name(), svc.Name(), ps)
		svc.Manager = sm
		svc.Transition(ps, func(et EventType) {
			counters.Add("events", 1)
			err = svc.EventHandler.Trigger(&Event{et, svc, nil})
			if err != nil {
				util.Warn("Error firing event: %s", err.Error())
			}
		})
		break
	}
	if svc.Manager == nil {
		return fmt.Errorf("Could not find service %s, did you misspell it?", svc.Name())
	}
	return nil
}
Esempio n. 6
0
func (ps *processStorage) AddSource(name string, config map[string]string) (Source, error) {
	for _, x := range ps.daemonSpecific {
		if x.Name() == name {
			return x, nil
		}
	}
	builder := Sources[name]
	if builder == nil {
		return nil, nil
	}
	util.Info("Activating metrics for %s", name)
	src, err := builder(config)
	if err != nil {
		return nil, err
	}

	m, ok := src.(MandatorySource)
	if ok && m.Mandatory() {
		util.Debug("Registering all metrics for %s", name)
		descs := src.ValidMetrics()
		for _, d := range descs {
			if d.MetricType == Counter {
				ps.DeclareCounter(name, d.Name, nil, d.Display)
			} else {
				ps.DeclareGauge(name, d.Name, d.Display)
			}
		}
	}

	ps.daemonSpecific = append(ps.daemonSpecific, src)
	return src, nil
}
Esempio n. 7
0
func sendSlackAlert(url string, params url.Values) error {
	util.Debug("Sending slack alert to %s", url)
	resp, err := http.PostForm(url, params)
	if resp != nil {
		resp.Body.Close()
	}
	return err
}
Esempio n. 8
0
func triggeredHandler(rule *Rule, tripped bool) *Event {
	if !tripped {
		util.Info("%s[%s] recovered.", rule.EntityName(), rule.Metric())
		rule.State = Recovered
		return nil
	}
	util.Debug("%s[%s] still triggered. Current: %.1f, Threshold: %.1f", rule.EntityName(), rule.Metric(), rule.CurrentValue, rule.Threshold)
	return nil
}
Esempio n. 9
0
func defaultClient(host string, port string, ep string) ([]byte, error) {
	url := fmt.Sprintf("http://%s:%s%s", host, port, ep)
	util.Debug("Fetching nginx status from %s", url)
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	return ioutil.ReadAll(resp.Body)
}
Esempio n. 10
0
func okHandler(rule *Rule, tripped bool) *Event {
	if tripped && rule.TrippedCount == rule.CycleCount {
		util.Warn("%s[%s] triggered.  Current value = %.1f", rule.EntityName(), rule.Metric(), rule.CurrentValue)
		rule.State = Triggered
		return &Event{RuleFailed, rule.Entity, rule}
	}
	if tripped {
		util.Debug("%s[%s] tripped. Current: %.1f, Threshold: %.1f", rule.EntityName(), rule.Metric(), rule.CurrentValue, rule.Threshold)
	}
	return nil
}
Esempio n. 11
0
func (i *Inspeqtor) Start() {
	util.Debug("Starting command socket")
	err := i.openSocket(i.SocketPath)
	if err != nil {
		util.Warn("Could not create Unix socket: %s", err.Error())
		exit(i)
	}

	go func() {
		for {
			if !i.safelyAccept() {
				util.Debug("Shutting down command socket")
				return
			}
		}
	}()

	// if expose_port is 0, disable the feature altogether
	if i.GlobalConfig.ExposePort != 0 {
		sock, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", i.GlobalConfig.ExposePort))
		if err != nil {
			util.Warn("Could not listen on port %d: %s", i.GlobalConfig.ExposePort, err.Error())
			exit(i)
		}
		i.Expose = sock
		go func() {
			// TODO How do we error handling here?
			util.Info("Expose now available at port %d", i.GlobalConfig.ExposePort)
			err := http.Serve(i.Expose, nil)
			// Don't log an "error" when we shut down normally and close the socket
			if err != nil && !strings.Contains(err.Error(), "use of closed network") {
				util.Warn("HTTP server error: %s", err.Error())
			}
		}()
	}

	util.Debug("Starting main run loop")
	go i.runLoop()

	Singleton = i
}
Esempio n. 12
0
func (svc *Service) Reload() error {
	go func() {
		util.Debug("Reloading %s", svc.Name())
		err := svc.Manager.Reload(svc.Name())
		if err != nil {
			util.Warn(err.Error())
		} else {
			util.DebugDebug("Reloaded %s", svc.Name())
		}
	}()
	return nil
}
Esempio n. 13
0
func Watch(i *inspeqtor.Inspeqtor, jobs map[string]*Job) {
	util.Debug("Starting recurring job watcher")
	go func() {
		for {
			untilNext := check(jobs)
			select {
			case <-i.Stopping:
				// reloading inspeqtor
				util.Debug("Shutting down recurring job watcher")
				return
			case <-runNotifier:
				// we just got notified a job ran,
				// verify we don't need to fire JobRan
			case <-time.After(untilNext + time.Minute):
				// a job is due at this point in time.
				// add an extra minute to allow for race conditions
				// and slow performance
			}
		}
	}()
}
Esempio n. 14
0
func HandleSignals() {
	signals := make(chan os.Signal)
	for k := range SignalHandlers {
		signal.Notify(signals, k)
	}

	for {
		sig := <-signals
		util.Debug("Received signal %d", sig)
		funk := SignalHandlers[sig]
		funk(Singleton)
	}
}
Esempio n. 15
0
func parseJobs(global *inspeqtor.ConfigFile, confDir string) (map[string]*Job, error) {
	util.Debug("Parsing jobs in " + confDir)
	files, err := filepath.Glob(confDir + "/jobs.d/*.inq")
	if err != nil {
		return nil, err
	}

	jobs := map[string]*Job{}

	for _, filename := range files {
		util.DebugDebug("Parsing " + filename)
		data, err := ioutil.ReadFile(filename)
		if err != nil {
			return nil, err
		}

		s := lexer.NewLexer([]byte(data))
		p := parser.NewParser()
		obj, err := p.Parse(s)
		if err != nil {
			util.Warn("Unable to parse " + filename + ": " + err.Error())
			continue
		}

		astcontent := obj.(*ast.Content)
		for _, astjob := range astcontent.Jobs {
			if _, ok := jobs[astjob.Name]; ok {
				return nil, fmt.Errorf("Duplicate job %s", astjob.Name)
			}

			j := New(astjob.Name, astjob.Interval, astcontent.Parameters)

			owner := j.Parameters["owner"]
			route := global.AlertRoutes[owner]
			if owner == "" && route == nil {
				return nil, fmt.Errorf("No default alert route configured!")
			}
			if route == nil {
				return nil, fmt.Errorf("No such alert route: %s", owner)
			}
			alert, err := inspeqtor.Actions["alert"](j, route)
			if err != nil {
				return nil, err
			}
			j.alerter = alert
			jobs[astjob.Name] = j
		}
	}

	return jobs, nil
}
Esempio n. 16
0
func (svc *Service) Restart() error {
	svc.Process.Pid = 0
	svc.Process.Status = services.Starting
	go func() {
		util.Debug("Restarting %s", svc.Name())
		err := svc.Manager.Restart(svc.Name())
		if err != nil {
			util.Warn(err.Error())
		} else {
			util.DebugDebug("Restarted %s", svc.Name())
		}
	}()
	return nil
}
Esempio n. 17
0
func sendEmail(e *EmailNotifier, doc bytes.Buffer) error {
	if strings.Index(e.To, "@example.com") > 0 {
		util.Warn("Invalid email configured: %s", e.To)
		util.Warn(string(doc.Bytes()))
	} else {
		util.Debug("Sending email to %s", e.To)
		util.Debug("Sending email:\n%s", string(doc.Bytes()))
		if e.Username != "" {
			auth := smtp.PlainAuth("", e.Username, e.Password, e.Host)
			err := smtp.SendMail(e.Host+":"+e.TLSPort, auth, e.From,
				[]string{e.To}, doc.Bytes())
			if err != nil {
				return err
			}
		} else {
			err := smtp.SendMail(e.Host+":25", nil, e.From, []string{e.To}, doc.Bytes())
			if err != nil {
				return err
			}
		}
	}
	return nil
}
Esempio n. 18
0
func detectUpstart(path string) (InitSystem, error) {
	result, err := util.FileExists(path)
	if err != nil {
		return nil, err
	}

	if !result {
		util.Debug("upstart not detected, no " + path)
		return nil, nil
	}

	matches, err := filepath.Glob(path + "/*.conf")
	if err != nil {
		return nil, err
	}

	if len(matches) > 0 {
		util.Info("Detected upstart in " + path)
		return &Upstart{path, nil}, nil
	}

	util.Debug("upstart not detected, empty " + path)
	return nil, nil
}
Esempio n. 19
0
func (i *Inspeqtor) TestAlertRoutes() int {
	bad := 0
	util.Info("Testing alert routes")
	for _, route := range i.GlobalConfig.AlertRoutes {
		nm := route.Name
		if nm == "" {
			nm = "default"
		}
		util.Debug("Creating notification for %s/%s", route.Channel, nm)
		notifier, err := Actions["alert"](i.Host, route)
		if err != nil {
			bad++
			util.Warn("Error creating %s/%s route: %s", route.Channel, nm, err.Error())
			continue
		}
		util.Debug("Triggering notification for %s/%s", route.Channel, nm)
		err = notifier.Trigger(&Event{RuleFailed, i.Host, i.Host.Rules()[0]})
		if err != nil {
			bad++
			util.Warn("Error firing %s/%s route: %s", route.Channel, nm, err.Error())
		}
	}
	return bad
}
Esempio n. 20
0
func detectSystemd(path string) (InitSystem, error) {
	result, err := util.FileExists(path)
	if err != nil {
		return nil, err
	}

	if !result {
		util.Debug("systemd not detected, no " + path)
		return nil, nil
	}

	matches, err := filepath.Glob(path + "/*.conf")
	if err != nil {
		return nil, err
	}

	if len(matches) > 0 {
		util.Info("Detected systemd in " + path)
		return &Systemd{path, "", ""}, nil
	}

	util.Debug("systemd not detected, empty " + path)
	return nil, nil
}
Esempio n. 21
0
func convertService(global *ConfigFile, inqsvc *ast.ProcessCheck) (*Service, error) {
	rules := make([]*Rule, len(inqsvc.Rules))
	storage := metrics.NewProcessStore("/proc", global.CycleTime)

	svc := &Service{&Entity{inqsvc.Name, nil, storage, inqsvc.Parameters}, nil, services.NewStatus(), nil}

	action, err := BuildAction(global, svc, &ast.SimpleAction{ActionName: "alert"})
	if err != nil {
		return nil, err
	}
	svc.EventHandler = action

	for idx, rule := range inqsvc.Rules {
		rule, err := convertRule(global, svc, rule)
		if err != nil {
			return nil, err
		}
		util.DebugDebug("Rule: %+v", *rule)
		rules[idx] = rule
	}
	svc.rules = rules

	for _, r := range rules {
		_, err := storage.AddSource(r.MetricFamily, svc.Parameters())
		if err != nil {
			return nil, err
		}

		err = storage.Watch(r.MetricFamily, r.MetricName)
		if err != nil {
			return nil, err
		}
		util.Debug("Watching %s:%s", r.MetricFamily, r.MetricName)
	}

	if len(inqsvc.Exposed) > 0 {
		err := BuildExpose(global, svc, inqsvc.Exposed, inqsvc.Parameters)
		if err != nil {
			return nil, err
		}
	}

	err = storage.Prepare()
	if err != nil {
		return nil, err
	}
	return svc, nil
}
Esempio n. 22
0
func ParseGlobal(rootDir string) (*ConfigFile, error) {
	path := rootDir + "/inspeqtor.conf"
	exists, err := util.FileExists(path)
	if err != nil {
		return nil, err
	}

	if exists {
		util.Debug("Parsing " + path)
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return nil, err
		}

		s := lexer.NewLexer([]byte(data))
		p := parser.NewParser()
		obj, err := p.Parse(s)
		if err != nil {
			return nil, err
		}
		ast := obj.(ast.Config)

		config := ConfigFile{Defaults, map[string]*AlertRoute{}}

		config.Variables = ast.Variables
		if val, has := ast.Variables["log_level"]; has {
			util.SetLogLevel(val)
		}
		parseValue(ast, &config.CycleTime, "cycle_time", 15)
		parseValue(ast, &config.DeployLength, "deploy_length", 300)
		parseValue(ast, &config.ExposePort, "expose_port", 4677)

		for _, v := range ast.Routes {
			ar, err := ValidateChannel(v.Name, v.Channel, v.Config)
			if err != nil {
				return nil, err
			}
			if _, ok := config.AlertRoutes[v.Name]; ok {
				return nil, fmt.Errorf("Duplicate alert config for '%s'", v.Name)
			}
			config.AlertRoutes[v.Name] = ar
		}
		return &config, nil
	}

	util.Info("No configuration file found at " + rootDir + "/inspector.conf")
	return &ConfigFile{Defaults, nil}, nil
}
Esempio n. 23
0
func (i *Inspeqtor) scan() {
	start := time.Now()
	var barrier sync.WaitGroup
	barrier.Add(1)
	barrier.Add(len(i.Services))

	go i.Host.Collect(i.silenced(), func(_ Checkable) {
		barrier.Done()
	})
	for _, svc := range i.Services {
		go svc.Collect(i.silenced(), func(_ Checkable) {
			barrier.Done()
		})
	}
	barrier.Wait()
	util.Debug("Collection complete in " + time.Now().Sub(start).String())
}
Esempio n. 24
0
func sendHipchatAlert(url, token string, msg url.Values) error {
	util.Debug("Sending hipchat alert to %s", url)

	client := &http.Client{}
	req, err := http.NewRequest("POST", url, strings.NewReader(msg.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	resp, err := client.Do(req)
	if resp != nil {
		if resp.StatusCode != 200 {
			util.Warn("Unable to send hipchat alert: %d", resp.StatusCode)
		}
		resp.Body.Close()
	}
	return err
}
Esempio n. 25
0
// this method never returns.
//
// since we can't test this method in an automated fashion, it should
// contain as little logic as possible.
func (i *Inspeqtor) runLoop() {
	util.DebugDebug("Resolving services")
	for _, svc := range i.Services {
		err := svc.Resolve(i.ServiceManagers)
		if err != nil {
			util.Warn(err.Error())
		}
	}

	i.scanSystem()

	for {
		select {
		case <-time.After(time.Duration(i.GlobalConfig.CycleTime) * time.Second):
			i.scanSystem()
		case <-i.Stopping:
			util.Debug("Shutting down main run loop")
			return
		}
	}
}
Esempio n. 26
0
func statsdReload(_ *inspeqtor.Inspeqtor, newi *inspeqtor.Inspeqtor) error {
	val, ok := newi.GlobalConfig.Variables["statsd_location"]
	if !ok {
		util.Debug("No statsd_location configured, skipping...")
		return nil
	}

	util.Info("Pushing metrics to statsd at %s", val)
	conn, err := statsd.Dial(val)
	if err != nil {
		return err
	}
	newi.Listen("cycleComplete", func(ins *inspeqtor.Inspeqtor) error {
		return statsd.Export(conn, ins)
	})
	newi.Listen("shutdown", func(ins *inspeqtor.Inspeqtor) error {
		return conn.Close()
	})

	return nil
}
Esempio n. 27
0
/*
Parses the service-specific rules in /etc/inspeqtor/services.d/*.inq
*/
func ParseServices(global *ConfigFile, confDir string) ([]Checkable, error) {
	util.Debug("Parsing config in " + confDir)
	files, err := filepath.Glob(confDir + "/*.inq")
	if err != nil {
		return nil, err
	}

	var checks []Checkable

	for _, filename := range files {
		util.DebugDebug("Parsing " + filename)
		data, err := ioutil.ReadFile(filename)
		if err != nil {
			return nil, err
		}

		s := lexer.NewLexer([]byte(data))
		p := parser.NewParser()
		obj, err := p.Parse(s)
		if err != nil {
			util.Warn("Unable to parse " + filename + ": " + err.Error())
			continue
		}

		switch x := obj.(type) {
		case *ast.ProcessCheck:
			svc, err := BuildService(global, x)
			if err != nil {
				return nil, err
			}
			util.DebugDebug("Service: %+v", *svc)
			checks = append(checks, svc)
		default:
			return nil, fmt.Errorf("Invalid configuration file: %s", filename)
		}
	}

	return checks, nil
}
Esempio n. 28
0
func (svc *Service) Verify() []*Event {
	events := []*Event{}

	if svc.Process.Status != services.Up {
		// we probably shouldn't verify anything that isn't actually Up
		util.Debug("%s is %s, skipping...", svc.Name(), svc.Process.Status)
		return events
	}

	for _, r := range svc.Rules() {
		evt := r.Check(svc.CycleTime())
		if evt != nil {
			events = append(events, evt)
			for _, a := range r.Actions {
				err := a.Trigger(evt)
				if err != nil {
					util.Warn("Error firing event: %s", err.Error())
				}
			}
		}
	}
	return events
}
Esempio n. 29
0
func sendCampfireAlert(url, token string, msg map[string]map[string]string) error {
	util.Debug("Sending campfire alert to %s", url)

	client := &http.Client{}
	jsonMsg, err := json.Marshal(msg)
	if err != nil {
		return err
	}
	req, err := http.NewRequest("POST", url, bytes.NewReader(jsonMsg))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("User-Agent", "Inspeqtor Pro")
	req.SetBasicAuth(token, "X")
	resp, err := client.Do(req)
	if resp != nil {
		if resp.StatusCode != 201 {
			util.Warn("Unable to send campfire alert: %d", resp.StatusCode)
		}
		resp.Body.Close()
	}
	return err
}