func readConf() *conf.Conf {
conf := &conf.Conf{
Freq: 15,
}
loc := *flagConf
if *flagConf == "" {
p, err := exePath()
if err != nil {
slog.Error(err)
return conf
}
dir := filepath.Dir(p)
loc = filepath.Join(dir, "scollector.toml")
}
f, err := os.Open(loc)
if err != nil {
if *flagConf != "" {
slog.Fatal(err)
}
if *flagDebug {
slog.Error(err)
}
} else {
defer f.Close()
md, err := toml.DecodeReader(f, conf)
if err != nil {
slog.Fatal(err)
}
if u := md.Undecoded(); len(u) > 0 {
slog.Fatalf("extra keys in %s: %v", loc, u)
}
}
return conf
}
func readConf() *Conf {
conf := &Conf{
Freq: 15,
}
loc := *flagConf
if *flagConf == "" {
p, err := exePath()
if err != nil {
slog.Error(err)
return conf
}
dir := filepath.Dir(p)
loc = filepath.Join(dir, "scollector.toml")
}
f, err := os.Open(loc)
if err != nil {
if *flagConf != "" {
slog.Fatal(err)
}
if *flagDebug {
slog.Error(err)
}
} else {
defer f.Close()
_, err := toml.DecodeReader(f, conf)
if err != nil {
slog.Fatal(err)
}
}
return conf
}
func watch(root, pattern string, f func()) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
slog.Fatal(err)
}
filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if matched, err := filepath.Match(pattern, info.Name()); err != nil {
slog.Fatal(err)
} else if !matched {
return nil
}
err = watcher.Add(path)
if err != nil {
slog.Fatal(err)
}
return nil
})
slog.Infoln("watching", pattern, "in", root)
wait := time.Now()
go func() {
for {
select {
case event := <-watcher.Events:
if wait.After(time.Now()) {
continue
}
if event.Op&fsnotify.Write == fsnotify.Write {
f()
wait = time.Now().Add(time.Second * 2)
}
case err := <-watcher.Errors:
slog.Errorln("error:", err)
}
}
}()
}
func startGithubCollectors(c *conf.Conf) {
for _, gh := range c.Github {
client := github.NewClient(&http.Client{Transport: githubRoundTripper{gh.Token}})
split := strings.Split(gh.Repo, "/")
if len(split) != 2 {
slog.Fatal("Repo must have two parts (owner/repo)")
}
owner, repo := split[0], split[1]
collectors = append(collectors, &IntervalCollector{
F: func() (opentsdb.MultiDataPoint, error) {
return githubCollect(client, owner, repo)
},
name: fmt.Sprintf("github-%s", gh.Repo),
Interval: 10 * time.Minute, //10 minutes to respect api limits
})
}
}
func main() {
flag.Parse()
if *flagVersion {
fmt.Println(version.GetVersionInfo("bosun"))
os.Exit(0)
}
for _, m := range mains {
m()
}
runtime.GOMAXPROCS(runtime.NumCPU())
c, err := conf.ParseFile(*flagConf)
if err != nil {
slog.Fatal(err)
}
if *flagTest {
os.Exit(0)
}
httpListen := &url.URL{
Scheme: "http",
Host: c.HTTPListen,
}
if strings.HasPrefix(httpListen.Host, ":") {
httpListen.Host = "localhost" + httpListen.Host
}
if err := metadata.Init(httpListen, false); err != nil {
slog.Fatal(err)
}
if err := sched.Load(c); err != nil {
slog.Fatal(err)
}
if c.RelayListen != "" {
go func() {
mux := http.NewServeMux()
mux.Handle("/api/", httputil.NewSingleHostReverseProxy(httpListen))
s := &http.Server{
Addr: c.RelayListen,
Handler: mux,
}
slog.Fatal(s.ListenAndServe())
}()
}
if c.TSDBHost != "" {
if err := collect.Init(httpListen, "bosun"); err != nil {
slog.Fatal(err)
}
tsdbHost := &url.URL{
Scheme: "http",
Host: c.TSDBHost,
}
if *flagReadonly {
rp := httputil.NewSingleHostReverseProxy(tsdbHost)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api/put" {
w.WriteHeader(204)
return
}
rp.ServeHTTP(w, r)
}))
slog.Infoln("readonly relay at", ts.URL, "to", tsdbHost)
tsdbHost, _ = url.Parse(ts.URL)
c.TSDBHost = tsdbHost.Host
}
}
if *flagQuiet {
c.Quiet = true
}
go func() { slog.Fatal(web.Listen(c.HTTPListen, *flagDev, c.TSDBHost)) }()
go func() {
if !*flagNoChecks {
sched.Run()
}
}()
go func() {
sc := make(chan os.Signal, 1)
signal.Notify(sc, os.Interrupt)
killing := false
for range sc {
if killing {
slog.Infoln("Second interrupt: exiting")
os.Exit(1)
}
killing = true
go func() {
slog.Infoln("Interrupt: closing down...")
sched.Close()
slog.Infoln("done")
os.Exit(1)
}()
}
}()
if *flagWatch {
watch(".", "*.go", quit)
watch(filepath.Join("web", "static", "templates"), "*.html", web.RunEsc)
base := filepath.Join("web", "static", "js")
watch(base, "*.ts", web.RunTsc)
}
select {}
}
func main() {
flag.Parse()
if *flagToToml != "" {
toToml(*flagToToml)
fmt.Println("toml conversion complete; remove all empty values by hand (empty strings, 0)")
return
}
if *flagPrint || *flagDebug {
slog.Set(&slog.StdLog{Log: log.New(os.Stdout, "", log.LstdFlags)})
}
if *flagVersion {
fmt.Println(version.GetVersionInfo("scollector"))
os.Exit(0)
}
for _, m := range mains {
m()
}
conf := readConf()
if *flagHost != "" {
conf.Host = *flagHost
}
if *flagFilter != "" {
conf.Filter = strings.Split(*flagFilter, ",")
}
if !conf.Tags.Valid() {
slog.Fatalf("invalid tags: %v", conf.Tags)
} else if conf.Tags["host"] != "" {
slog.Fatalf("host not supported in custom tags, use Hostname instead")
}
if conf.PProf != "" {
go func() {
slog.Infof("Starting pprof at http://%s/debug/pprof/", conf.PProf)
slog.Fatal(http.ListenAndServe(conf.PProf, nil))
}()
}
collectors.AddTags = conf.Tags
util.FullHostname = conf.FullHost
util.Set()
if conf.Hostname != "" {
util.Hostname = conf.Hostname
}
if err := collect.SetHostname(util.Hostname); err != nil {
slog.Fatal(err)
}
if conf.ColDir != "" {
collectors.InitPrograms(conf.ColDir)
}
var err error
check := func(e error) {
if e != nil {
err = e
}
}
collectors.Init(conf)
for _, r := range conf.MetricFilters {
check(collectors.AddMetricFilters(r))
}
for _, rmq := range conf.RabbitMQ {
check(collectors.RabbitMQ(rmq.URL))
}
for _, cfg := range conf.SNMP {
check(collectors.SNMP(cfg, conf.MIBS))
}
for _, i := range conf.ICMP {
check(collectors.ICMP(i.Host))
}
for _, a := range conf.AWS {
check(collectors.AWS(a.AccessKey, a.SecretKey, a.Region))
}
for _, v := range conf.Vsphere {
check(collectors.Vsphere(v.User, v.Password, v.Host))
}
for _, p := range conf.Process {
check(collectors.AddProcessConfig(p))
}
for _, p := range conf.ProcessDotNet {
check(collectors.AddProcessDotNetConfig(p))
}
for _, h := range conf.HTTPUnit {
if h.TOML != "" {
check(collectors.HTTPUnitTOML(h.TOML))
}
if h.Hiera != "" {
check(collectors.HTTPUnitHiera(h.Hiera))
}
}
for _, r := range conf.Riak {
check(collectors.Riak(r.URL))
}
for _, x := range conf.ExtraHop {
check(collectors.ExtraHop(x.Host, x.APIKey, x.FilterBy, x.FilterPercent))
}
if err != nil {
slog.Fatal(err)
}
collectors.KeepalivedCommunity = conf.KeepalivedCommunity
// Add all process collectors. This is platform specific.
collectors.WatchProcesses()
collectors.WatchProcessesDotNet()
if *flagFake > 0 {
collectors.InitFake(*flagFake)
}
collect.Debug = *flagDebug
util.Debug = *flagDebug
collect.DisableDefaultCollectors = conf.DisableSelf
c := collectors.Search(conf.Filter)
if len(c) == 0 {
slog.Fatalf("Filter %v matches no collectors.", conf.Filter)
}
for _, col := range c {
col.Init()
}
u, err := parseHost(conf.Host)
if *flagList {
list(c)
return
} else if *flagPrint {
u = &url.URL{Scheme: "http", Host: "localhost:0"}
} else if err != nil {
slog.Fatalf("invalid host %v: %v", conf.Host, err)
}
freq := time.Second * time.Duration(conf.Freq)
if freq <= 0 {
slog.Fatal("freq must be > 0")
}
collectors.DefaultFreq = freq
collect.Freq = freq
if conf.BatchSize < 0 {
slog.Fatal("BatchSize must be > 0")
}
if conf.BatchSize != 0 {
collect.BatchSize = conf.BatchSize
}
collect.Tags = conf.Tags.Copy().Merge(opentsdb.TagSet{"os": runtime.GOOS})
if *flagPrint {
collect.Print = true
}
if !*flagDisableMetadata {
if err := metadata.Init(u, *flagDebug); err != nil {
slog.Fatal(err)
}
}
cdp, cquit := collectors.Run(c)
if u != nil {
slog.Infoln("OpenTSDB host:", u)
}
if err := collect.InitChan(u, "scollector", cdp); err != nil {
slog.Fatal(err)
}
if version.VersionDate != "" {
v, err := strconv.ParseInt(version.VersionDate, 10, 64)
if err == nil {
go func() {
metadata.AddMetricMeta("scollector.version", metadata.Gauge, metadata.None,
"Scollector version number, which indicates when scollector was built.")
for {
if err := collect.Put("version", collect.Tags, v); err != nil {
slog.Error(err)
}
time.Sleep(time.Hour)
}
}()
}
}
if *flagBatchSize > 0 {
collect.BatchSize = *flagBatchSize
}
go func() {
const maxMem = 500 * 1024 * 1024 // 500MB
var m runtime.MemStats
for range time.Tick(time.Minute) {
runtime.ReadMemStats(&m)
if m.Alloc > maxMem {
panic("memory max reached")
}
}
}()
sChan := make(chan os.Signal)
signal.Notify(sChan, os.Interrupt)
<-sChan
close(cquit)
// try to flush all datapoints on sigterm, but quit after 5 seconds no matter what.
time.AfterFunc(5*time.Second, func() {
os.Exit(0)
})
collect.Flush()
}
func toToml(fname string) {
var c conf.Conf
b, err := ioutil.ReadFile(*flagConf)
if err != nil {
slog.Fatal(err)
}
extra := new(bytes.Buffer)
var hap conf.HAProxy
for i, line := range strings.Split(string(b), "\n") {
if strings.TrimSpace(line) == "" {
continue
}
sp := strings.SplitN(line, "=", 2)
if len(sp) != 2 {
slog.Fatalf("expected = in %v:%v", *flagConf, i+1)
}
k := strings.TrimSpace(sp[0])
v := strings.TrimSpace(sp[1])
switch k {
case "host":
c.Host = v
case "hostname":
c.Hostname = v
case "filter":
c.Filter = strings.Split(v, ",")
case "coldir":
c.ColDir = v
case "snmp":
for _, s := range strings.Split(v, ",") {
sp := strings.Split(s, "@")
if len(sp) != 2 {
slog.Fatal("invalid snmp string:", v)
}
c.SNMP = append(c.SNMP, conf.SNMP{
Community: sp[0],
Host: sp[1],
})
}
case "icmp":
for _, i := range strings.Split(v, ",") {
c.ICMP = append(c.ICMP, conf.ICMP{Host: i})
}
case "haproxy":
if v != "" {
for _, s := range strings.Split(v, ",") {
sp := strings.SplitN(s, ":", 2)
if len(sp) != 2 {
slog.Fatal("invalid haproxy string:", v)
}
if hap.User != "" || hap.Password != "" {
slog.Fatal("only one haproxy line allowed")
}
hap.User = sp[0]
hap.Password = sp[1]
}
}
case "haproxy_instance":
sp := strings.SplitN(v, ":", 2)
if len(sp) != 2 {
slog.Fatal("invalid haproxy_instance string:", v)
}
hap.Instances = append(hap.Instances, conf.HAProxyInstance{
Tier: sp[0],
URL: sp[1],
})
case "tags":
tags, err := opentsdb.ParseTags(v)
if err != nil {
slog.Fatal(err)
}
c.Tags = tags
case "aws":
for _, s := range strings.Split(v, ",") {
sp := strings.SplitN(s, ":", 2)
if len(sp) != 2 {
slog.Fatal("invalid AWS string:", v)
}
accessKey := sp[0]
idx := strings.LastIndex(sp[1], "@")
if idx == -1 {
slog.Fatal("invalid AWS string:", v)
}
secretKey := sp[1][:idx]
region := sp[1][idx+1:]
if len(accessKey) == 0 || len(secretKey) == 0 || len(region) == 0 {
slog.Fatal("invalid AWS string:", v)
}
c.AWS = append(c.AWS, conf.AWS{
AccessKey: accessKey,
SecretKey: secretKey,
Region: region,
})
}
case "vsphere":
for _, s := range strings.Split(v, ",") {
sp := strings.SplitN(s, ":", 2)
if len(sp) != 2 {
slog.Fatal("invalid vsphere string:", v)
}
user := sp[0]
idx := strings.LastIndex(sp[1], "@")
if idx == -1 {
slog.Fatal("invalid vsphere string:", v)
}
pwd := sp[1][:idx]
host := sp[1][idx+1:]
if len(user) == 0 || len(pwd) == 0 || len(host) == 0 {
slog.Fatal("invalid vsphere string:", v)
}
c.Vsphere = append(c.Vsphere, conf.Vsphere{
User: user,
Password: pwd,
Host: host,
})
}
case "freq":
freq, err := strconv.Atoi(v)
if err != nil {
slog.Fatal(err)
}
c.Freq = freq
case "process":
if runtime.GOOS == "linux" {
var p struct {
Command string
Name string
Args string
}
sp := strings.Split(v, ",")
if len(sp) > 1 {
p.Name = sp[1]
}
if len(sp) > 2 {
p.Args = sp[2]
}
p.Command = sp[0]
extra.WriteString(fmt.Sprintf(`
[[Process]]
Command = %q
Name = %q
Args = %q
`, p.Command, p.Name, p.Args))
} else if runtime.GOOS == "windows" {
extra.WriteString(fmt.Sprintf(`
[[Process]]
Name = %q
`, v))
}
case "process_dotnet":
c.ProcessDotNet = append(c.ProcessDotNet, conf.ProcessDotNet{Name: v})
case "keepalived_community":
c.KeepalivedCommunity = v
default:
slog.Fatalf("unknown key in %v:%v", *flagConf, i+1)
}
}
if len(hap.Instances) > 0 {
c.HAProxy = append(c.HAProxy, hap)
}
f, err := os.Create(fname)
if err != nil {
slog.Fatal(err)
}
if err := toml.NewEncoder(f).Encode(&c); err != nil {
slog.Fatal(err)
}
if _, err := extra.WriteTo(f); err != nil {
slog.Fatal(err)
}
f.Close()
}
func main() {
flag.Parse()
if *flagToToml != "" {
toToml(*flagToToml)
fmt.Println("toml conversion complete; remove all empty values by hand (empty strings, 0)")
return
}
if *flagPrint || *flagDebug {
slog.Set(&slog.StdLog{Log: log.New(os.Stdout, "", log.LstdFlags)})
}
if *flagVersion {
fmt.Println(version.GetVersionInfo("scollector"))
os.Exit(0)
}
for _, m := range mains {
m()
}
conf := readConf()
ua := "Scollector/" + version.ShortVersion()
if conf.UserAgentMessage != "" {
ua += fmt.Sprintf(" (%s)", conf.UserAgentMessage)
}
client := &http.Client{
Transport: &scollectorHTTPTransport{
ua,
&httpcontrol.Transport{
RequestTimeout: time.Minute,
},
},
}
http.DefaultClient = client
collect.DefaultClient = client
if *flagHost != "" {
conf.Host = *flagHost
}
if *flagNtlm {
conf.UseNtlm = *flagNtlm
}
if *flagFilter != "" {
conf.Filter = strings.Split(*flagFilter, ",")
}
if !conf.Tags.Valid() {
slog.Fatalf("invalid tags: %v", conf.Tags)
} else if conf.Tags["host"] != "" {
slog.Fatalf("host not supported in custom tags, use Hostname instead")
}
if conf.PProf != "" {
go func() {
slog.Infof("Starting pprof at http://%s/debug/pprof/", conf.PProf)
slog.Fatal(http.ListenAndServe(conf.PProf, nil))
}()
}
collectors.AddTags = conf.Tags
util.FullHostname = conf.FullHost
util.Set()
if conf.Hostname != "" {
util.Hostname = conf.Hostname
}
if err := collect.SetHostname(util.Hostname); err != nil {
slog.Fatal(err)
}
if conf.ColDir != "" {
collectors.InitPrograms(conf.ColDir)
}
if conf.SNMPTimeout > 0 {
snmp.Timeout = conf.SNMPTimeout
}
var err error
check := func(e error) {
if e != nil {
err = e
}
}
collectors.Init(conf)
for _, r := range conf.MetricFilters {
slog.Infof("Adding MetricFilter: %v\n", r)
check(collectors.AddMetricFilters(r))
}
for _, rmq := range conf.RabbitMQ {
check(collectors.RabbitMQ(rmq.URL))
}
for _, cfg := range conf.SNMP {
check(collectors.SNMP(cfg, conf.MIBS))
}
for _, i := range conf.ICMP {
check(collectors.ICMP(i.Host))
}
for _, a := range conf.AWS {
check(collectors.AWS(a.AccessKey, a.SecretKey, a.Region, a.BillingProductCodesRegex, a.BillingBucketName, a.BillingBucketPath, a.BillingPurgeDays))
}
for _, ea := range conf.AzureEA {
check(collectors.AzureEABilling(ea.EANumber, ea.APIKey, ea.LogBillingDetails))
}
for _, v := range conf.Vsphere {
check(collectors.Vsphere(v.User, v.Password, v.Host))
}
for _, p := range conf.Process {
check(collectors.AddProcessConfig(p))
}
for _, p := range conf.ProcessDotNet {
check(collectors.AddProcessDotNetConfig(p))
}
for _, h := range conf.HTTPUnit {
var freq time.Duration
var parseerr error
if h.Freq == "" {
freq = time.Minute * 5
} else {
freq, parseerr = time.ParseDuration(h.Freq)
if parseerr != nil {
slog.Fatal(parseerr)
}
if freq < time.Second {
slog.Fatalf("Invalid HTTPUnit frequency %s, cannot be less than 1 second.", h.Freq)
}
}
if h.TOML != "" {
check(collectors.HTTPUnitTOML(h.TOML, freq))
}
if h.Hiera != "" {
check(collectors.HTTPUnitHiera(h.Hiera, freq))
}
}
for _, r := range conf.Riak {
check(collectors.Riak(r.URL))
}
for _, x := range conf.ExtraHop {
check(collectors.ExtraHop(x.Host, x.APIKey, x.FilterBy, x.FilterPercent, x.AdditionalMetrics, x.CertificateSubjectMatch, x.CertificateActivityGroup))
}
if err != nil {
slog.Fatal(err)
}
collectors.KeepalivedCommunity = conf.KeepalivedCommunity
// Add all process collectors. This is platform specific.
collectors.WatchProcesses()
collectors.WatchProcessesDotNet()
if *flagFake > 0 {
collectors.InitFake(*flagFake)
}
collect.Debug = *flagDebug
util.Debug = *flagDebug
collect.DisableDefaultCollectors = conf.DisableSelf
c := collectors.Search(conf.Filter)
if len(c) == 0 {
slog.Fatalf("Filter %v matches no collectors.", conf.Filter)
}
for _, col := range c {
col.Init()
}
err = collectors.AddTagOverrides(c, conf.TagOverride)
if err != nil {
slog.Fatalf("Error adding tag overrides: %s", err)
}
u, err := parseHost(conf.Host)
if *flagList {
list(c)
return
} else if *flagPrint {
u = &url.URL{Scheme: "http", Host: "localhost:0"}
} else if err != nil {
slog.Fatalf("invalid host %v: %v", conf.Host, err)
}
freq := time.Second * time.Duration(conf.Freq)
if freq <= 0 {
slog.Fatal("freq must be > 0")
}
collectors.DefaultFreq = freq
collect.Freq = freq
if conf.BatchSize < 0 {
slog.Fatal("BatchSize must be > 0")
}
if conf.BatchSize != 0 {
collect.BatchSize = conf.BatchSize
}
collect.Tags = conf.Tags.Copy().Merge(opentsdb.TagSet{"os": runtime.GOOS})
if *flagPrint {
collect.Print = true
}
if !*flagDisableMetadata {
if err := metadata.Init(u, *flagDebug); err != nil {
slog.Fatal(err)
}
}
cdp, cquit := collectors.Run(c)
if u != nil {
slog.Infoln("OpenTSDB host:", u)
}
collect.UseNtlm = conf.UseNtlm
if err := collect.InitChan(u, "scollector", cdp); err != nil {
slog.Fatal(err)
}
if collect.DisableDefaultCollectors == false && version.VersionDate != "" {
v, err := strconv.ParseInt(version.VersionDate, 10, 64)
if err == nil {
go func() {
metadata.AddMetricMeta("scollector.version", metadata.Gauge, metadata.None,
"Scollector version number, which indicates when scollector was built.")
for {
if err := collect.Put("version", collect.Tags, v); err != nil {
slog.Error(err)
}
time.Sleep(time.Hour)
}
}()
}
}
if *flagBatchSize > 0 {
collect.BatchSize = *flagBatchSize
}
if conf.MaxQueueLen != 0 {
if conf.MaxQueueLen < collect.BatchSize {
slog.Fatalf("MaxQueueLen must be >= %d (BatchSize)", collect.BatchSize)
}
collect.MaxQueueLen = conf.MaxQueueLen
}
maxMemMB := uint64(500)
if conf.MaxMem != 0 {
maxMemMB = conf.MaxMem
}
go func() {
var m runtime.MemStats
for range time.Tick(time.Second * 30) {
runtime.ReadMemStats(&m)
allocMB := m.Alloc / 1024 / 1024
if allocMB > maxMemMB {
slog.Fatalf("memory max runtime reached: (current alloc: %v megabytes, max: %v megabytes)", allocMB, maxMemMB)
}
//See proccess_windows.go and process_linux.go for total process memory usage.
//Note that in linux the rss metric includes shared pages, where as in
//Windows the private working set does not include shared memory.
//Total memory used seems to scale linerarly with m.Alloc.
//But we want this to catch a memory leak outside the runtime (WMI/CGO).
//So for now just add any runtime allocations to the allowed total limit.
maxMemTotalMB := maxMemMB + allocMB
if collectors.TotalScollectorMemoryMB > maxMemTotalMB {
slog.Fatalf("memory max total reached: (current total: %v megabytes, current runtime alloc: %v megabytes, max: %v megabytes)", collectors.TotalScollectorMemoryMB, allocMB, maxMemTotalMB)
}
}
}()
sChan := make(chan os.Signal)
signal.Notify(sChan, os.Interrupt)
<-sChan
close(cquit)
// try to flush all datapoints on sigterm, but quit after 5 seconds no matter what.
time.AfterFunc(5*time.Second, func() {
os.Exit(0)
})
collect.Flush()
}
func main() {
flag.Parse()
if *flagToToml != "" {
toToml(*flagToToml)
fmt.Println("toml conversion complete; remove all empty values by hand (empty strings, 0)")
return
}
if *flagPrint || *flagDebug {
slog.Set(&slog.StdLog{Log: log.New(os.Stdout, "", log.LstdFlags)})
}
if *flagVersion {
fmt.Println(version.GetVersionInfo("scollector"))
os.Exit(0)
}
for _, m := range mains {
m()
}
conf := readConf()
if *flagHost != "" {
conf.Host = *flagHost
}
if *flagFilter != "" {
conf.Filter = strings.Split(*flagFilter, ",")
}
if !conf.Tags.Valid() {
slog.Fatalf("invalid tags: %v", conf.Tags)
} else if conf.Tags["host"] != "" {
slog.Fatalf("host not supported in custom tags, use Hostname instead")
}
collectors.AddTags = conf.Tags
util.FullHostname = conf.FullHost
util.Set()
if conf.Hostname != "" {
util.Hostname = conf.Hostname
if err := collect.SetHostname(conf.Hostname); err != nil {
slog.Fatal(err)
}
}
if conf.ColDir != "" {
collectors.InitPrograms(conf.ColDir)
}
var err error
check := func(e error) {
if e != nil {
err = e
}
}
for _, h := range conf.HAProxy {
for _, i := range h.Instances {
collectors.HAProxy(h.User, h.Password, i.Tier, i.URL)
}
}
for _, s := range conf.SNMP {
check(collectors.SNMP(s.Community, s.Host))
}
for _, i := range conf.ICMP {
check(collectors.ICMP(i.Host))
}
for _, a := range conf.AWS {
check(collectors.AWS(a.AccessKey, a.SecretKey, a.Region))
}
for _, v := range conf.Vsphere {
check(collectors.Vsphere(v.User, v.Password, v.Host))
}
for _, p := range conf.Process {
check(collectors.AddProcessConfig(p))
}
for _, h := range conf.HTTPUnit {
if h.TOML != "" {
check(collectors.HTTPUnitTOML(h.TOML))
}
if h.Hiera != "" {
check(collectors.HTTPUnitHiera(h.Hiera))
}
}
if err != nil {
slog.Fatal(err)
}
collectors.KeepalivedCommunity = conf.KeepalivedCommunity
// Add all process collectors. This is platform specific.
collectors.WatchProcesses()
collectors.WatchProcessesDotNet()
if *flagFake > 0 {
collectors.InitFake(*flagFake)
}
collect.Debug = *flagDebug
util.Debug = *flagDebug
collect.DisableDefaultCollectors = conf.DisableSelf
c := collectors.Search(conf.Filter)
if len(c) == 0 {
slog.Fatalf("Filter %v matches no collectors.", conf.Filter)
}
for _, col := range c {
col.Init()
}
u, err := parseHost(conf.Host)
if *flagList {
list(c)
return
} else if err != nil {
slog.Fatalf("invalid host %v: %v", conf.Host, err)
}
freq := time.Second * time.Duration(conf.Freq)
if freq <= 0 {
slog.Fatal("freq must be > 0")
}
collectors.DefaultFreq = freq
collect.Freq = freq
collect.Tags = opentsdb.TagSet{"os": runtime.GOOS}
if *flagPrint {
collect.Print = true
}
if !*flagDisableMetadata {
if err := metadata.Init(u, *flagDebug); err != nil {
slog.Fatal(err)
}
}
cdp := collectors.Run(c)
if u != nil {
slog.Infoln("OpenTSDB host:", u)
}
if err := collect.InitChan(u, "scollector", cdp); err != nil {
slog.Fatal(err)
}
if version.VersionDate != "" {
v, err := strconv.ParseInt(version.VersionDate, 10, 64)
if err == nil {
go func() {
metadata.AddMetricMeta("scollector.version", metadata.Gauge, metadata.None,
"Scollector version number, which indicates when scollector was built.")
for {
if err := collect.Put("version", collect.Tags, v); err != nil {
slog.Error(err)
}
time.Sleep(time.Hour)
}
}()
}
}
if *flagBatchSize > 0 {
collect.BatchSize = *flagBatchSize
}
go func() {
const maxMem = 500 * 1024 * 1024 // 500MB
var m runtime.MemStats
for range time.Tick(time.Minute) {
runtime.ReadMemStats(&m)
if m.Alloc > maxMem {
panic("memory max reached")
}
}
}()
select {}
}
func Listen(listenAddr string, devMode bool, tsdbHost string) error {
if devMode {
slog.Infoln("using local web assets")
}
webFS := FS(devMode)
indexTemplate = func() *template.Template {
str := FSMustString(devMode, "/templates/index.html")
templates, err := template.New("").Parse(str)
if err != nil {
slog.Fatal(err)
}
return templates
}
if !devMode {
tpl := indexTemplate()
indexTemplate = func() *template.Template {
return tpl
}
}
if tsdbHost != "" {
router.HandleFunc("/api/index", IndexTSDB)
router.Handle("/api/put", Relay(tsdbHost))
}
router.HandleFunc("/api/", APIRedirect)
router.Handle("/api/action", JSON(Action))
router.Handle("/api/alerts", JSON(Alerts))
router.Handle("/api/backup", JSON(Backup))
router.Handle("/api/config", miniprofiler.NewHandler(Config))
router.Handle("/api/config_test", miniprofiler.NewHandler(ConfigTest))
router.Handle("/api/egraph/{bs}.svg", JSON(ExprGraph))
router.Handle("/api/errors", JSON(ErrorHistory))
router.Handle("/api/expr", JSON(Expr))
router.Handle("/api/graph", JSON(Graph))
router.Handle("/api/health", JSON(HealthCheck))
router.Handle("/api/host", JSON(Host))
router.Handle("/api/last", JSON(Last))
router.Handle("/api/incidents", JSON(Incidents))
router.Handle("/api/incidents/events", JSON(IncidentEvents))
router.Handle("/api/metadata/get", JSON(GetMetadata))
router.Handle("/api/metadata/metrics", JSON(MetadataMetrics))
router.Handle("/api/metadata/put", JSON(PutMetadata))
router.Handle("/api/metadata/delete", JSON(DeleteMetadata)).Methods("DELETE")
router.Handle("/api/metric", JSON(UniqueMetrics))
router.Handle("/api/metric/{tagk}/{tagv}", JSON(MetricsByTagPair))
router.Handle("/api/rule", JSON(Rule))
router.HandleFunc("/api/shorten", Shorten)
router.Handle("/api/silence/clear", JSON(SilenceClear))
router.Handle("/api/silence/get", JSON(SilenceGet))
router.Handle("/api/silence/set", JSON(SilenceSet))
router.Handle("/api/status", JSON(Status))
router.Handle("/api/tagk/{metric}", JSON(TagKeysByMetric))
router.Handle("/api/tagv/{tagk}", JSON(TagValuesByTagKey))
router.Handle("/api/tagv/{tagk}/{metric}", JSON(TagValuesByMetricTagKey))
router.Handle("/api/tagsets/{metric}", JSON(FilteredTagsetsByMetric))
router.Handle("/api/opentsdb/version", JSON(OpenTSDBVersion))
router.HandleFunc("/api/version", Version)
router.Handle("/api/debug/schedlock", JSON(ScheduleLockStatus))
http.Handle("/", miniprofiler.NewHandler(Index))
http.Handle("/api/", router)
fs := http.FileServer(webFS)
http.Handle("/partials/", fs)
http.Handle("/static/", http.StripPrefix("/static/", fs))
http.Handle("/favicon.ico", fs)
slog.Infoln("bosun web listening on:", listenAddr)
slog.Infoln("tsdb host:", tsdbHost)
return http.ListenAndServe(listenAddr, nil)
}
func Listen(listenAddr string, devMode bool, tsdbHost string, reloadFunc func() error) error {
if devMode {
slog.Infoln("using local web assets")
}
webFS := FS(devMode)
indexTemplate = func() *template.Template {
str := FSMustString(devMode, "/templates/index.html")
templates, err := template.New("").Parse(str)
if err != nil {
slog.Fatal(err)
}
return templates
}
reload = reloadFunc
if !devMode {
tpl := indexTemplate()
indexTemplate = func() *template.Template {
return tpl
}
}
if tsdbHost != "" {
router.HandleFunc("/api/index", IndexTSDB)
router.Handle("/api/put", Relay(tsdbHost))
}
router.HandleFunc("/api/", APIRedirect)
router.Handle("/api/action", JSON(Action))
router.Handle("/api/alerts", JSON(Alerts))
router.Handle("/api/config", miniprofiler.NewHandler(Config))
router.Handle("/api/config_test", miniprofiler.NewHandler(ConfigTest))
router.Handle("/api/save_enabled", JSON(SaveEnabled))
if schedule.SystemConf.ReloadEnabled() { // Is true of save is enabled
router.Handle("/api/reload", JSON(Reload)).Methods(http.MethodPost)
}
if schedule.SystemConf.SaveEnabled() {
router.Handle("/api/config/bulkedit", miniprofiler.NewHandler(BulkEdit)).Methods(http.MethodPost)
router.Handle("/api/config/save", miniprofiler.NewHandler(SaveConfig)).Methods(http.MethodPost)
router.Handle("/api/config/diff", miniprofiler.NewHandler(DiffConfig)).Methods(http.MethodPost)
router.Handle("/api/config/running_hash", JSON(ConfigRunningHash))
}
router.Handle("/api/egraph/{bs}.{format:svg|png}", JSON(ExprGraph))
router.Handle("/api/errors", JSON(ErrorHistory))
router.Handle("/api/expr", JSON(Expr))
router.Handle("/api/graph", JSON(Graph))
router.Handle("/api/health", JSON(HealthCheck))
router.Handle("/api/host", JSON(Host))
router.Handle("/api/last", JSON(Last))
router.Handle("/api/quiet", JSON(Quiet))
router.Handle("/api/incidents", JSON(Incidents))
router.Handle("/api/incidents/open", JSON(ListOpenIncidents))
router.Handle("/api/incidents/events", JSON(IncidentEvents))
router.Handle("/api/metadata/get", JSON(GetMetadata))
router.Handle("/api/metadata/metrics", JSON(MetadataMetrics))
router.Handle("/api/metadata/put", JSON(PutMetadata))
router.Handle("/api/metadata/delete", JSON(DeleteMetadata)).Methods("DELETE")
router.Handle("/api/metric", JSON(UniqueMetrics))
router.Handle("/api/metric/{tagk}", JSON(MetricsByTagKey))
router.Handle("/api/metric/{tagk}/{tagv}", JSON(MetricsByTagPair))
router.Handle("/api/rule", JSON(Rule))
router.HandleFunc("/api/shorten", Shorten)
router.Handle("/api/silence/clear", JSON(SilenceClear))
router.Handle("/api/silence/get", JSON(SilenceGet))
router.Handle("/api/silence/set", JSON(SilenceSet))
router.Handle("/api/status", JSON(Status))
router.Handle("/api/tagk/{metric}", JSON(TagKeysByMetric))
router.Handle("/api/tagv/{tagk}", JSON(TagValuesByTagKey))
router.Handle("/api/tagv/{tagk}/{metric}", JSON(TagValuesByMetricTagKey))
router.Handle("/api/tagsets/{metric}", JSON(FilteredTagsetsByMetric))
router.Handle("/api/opentsdb/version", JSON(OpenTSDBVersion))
router.Handle("/api/annotate", JSON(AnnotateEnabled))
// Annotations
if schedule.SystemConf.AnnotateEnabled() {
index := schedule.SystemConf.GetAnnotateIndex()
if index == "" {
index = "annotate"
}
annotateBackend = backend.NewElastic(schedule.SystemConf.GetAnnotateElasticHosts(), index)
go func() {
for {
err := annotateBackend.InitBackend()
if err == nil {
return
}
slog.Warningf("could not initalize annotate backend, will try again: %v", err)
time.Sleep(time.Second * 30)
}
}()
web.AddRoutes(router, "/api", []backend.Backend{annotateBackend}, false, false)
}
router.HandleFunc("/api/version", Version)
router.Handle("/api/debug/schedlock", JSON(ScheduleLockStatus))
http.Handle("/", miniprofiler.NewHandler(Index))
http.Handle("/api/", router)
fs := http.FileServer(webFS)
http.Handle("/partials/", fs)
http.Handle("/static/", http.StripPrefix("/static/", fs))
http.Handle("/favicon.ico", fs)
slog.Infoln("bosun web listening on:", listenAddr)
slog.Infoln("tsdb host:", tsdbHost)
return http.ListenAndServe(listenAddr, nil)
}
func main() {
flag.Parse()
if *flagVersion {
fmt.Println(version.GetVersionInfo("bosun"))
os.Exit(0)
}
for _, m := range mains {
m()
}
systemConf, err := conf.LoadSystemConfigFile(*flagConf)
if err != nil {
slog.Fatalf("couldn't read system configuration: %v", err)
}
sysProvider, err := systemConf.GetSystemConfProvider()
if err != nil {
slog.Fatal(err)
}
ruleConf, err := rule.ParseFile(sysProvider.GetRuleFilePath(), systemConf.EnabledBackends())
if err != nil {
slog.Fatalf("couldn't read rules: %v", err)
}
if *flagTest {
os.Exit(0)
}
var ruleProvider conf.RuleConfProvider = ruleConf
httpListen := &url.URL{
Scheme: "http",
Host: sysProvider.GetHTTPListen(),
}
if strings.HasPrefix(httpListen.Host, ":") {
httpListen.Host = "localhost" + httpListen.Host
}
if err := metadata.Init(httpListen, false); err != nil {
slog.Fatal(err)
}
if err := sched.Load(sysProvider, ruleProvider, *flagSkipLast, *flagQuiet); err != nil {
slog.Fatal(err)
}
if sysProvider.GetRelayListen() != "" {
go func() {
mux := http.NewServeMux()
mux.Handle("/api/", util.NewSingleHostProxy(httpListen))
s := &http.Server{
Addr: sysProvider.GetRelayListen(),
Handler: mux,
}
slog.Fatal(s.ListenAndServe())
}()
}
if sysProvider.GetTSDBHost() != "" {
if err := collect.Init(httpListen, "bosun"); err != nil {
slog.Fatal(err)
}
tsdbHost := &url.URL{
Scheme: "http",
Host: sysProvider.GetTSDBHost(),
}
if *flagReadonly {
rp := util.NewSingleHostProxy(tsdbHost)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api/put" {
w.WriteHeader(204)
return
}
rp.ServeHTTP(w, r)
}))
slog.Infoln("readonly relay at", ts.URL, "to", tsdbHost)
tsdbHost, _ = url.Parse(ts.URL)
sysProvider.SetTSDBHost(tsdbHost.Host)
}
}
if systemConf.GetPing() {
go ping.PingHosts(sched.DefaultSched.Search, systemConf.GetPingDuration())
}
if sysProvider.GetInternetProxy() != "" {
web.InternetProxy, err = url.Parse(sysProvider.GetInternetProxy())
if err != nil {
slog.Fatalf("InternetProxy error: %s", err)
}
}
var cmdHook conf.SaveHook
if hookPath := sysProvider.GetCommandHookPath(); hookPath != "" {
cmdHook, err = conf.MakeSaveCommandHook(hookPath)
if err != nil {
slog.Fatal(err)
}
ruleProvider.SetSaveHook(cmdHook)
}
var reload func() error
reloading := make(chan bool, 1) // a lock that we can give up acquiring
reload = func() error {
select {
case reloading <- true:
// Got lock
default:
return fmt.Errorf("not reloading, reload in progress")
}
defer func() {
<-reloading
}()
newConf, err := rule.ParseFile(sysProvider.GetRuleFilePath(), sysProvider.EnabledBackends())
if err != nil {
return err
}
newConf.SetSaveHook(cmdHook)
newConf.SetReload(reload)
oldSched := sched.DefaultSched
oldDA := oldSched.DataAccess
oldSearch := oldSched.Search
sched.Close(true)
sched.Reset()
newSched := sched.DefaultSched
newSched.Search = oldSearch
newSched.DataAccess = oldDA
slog.Infoln("schedule shutdown, loading new schedule")
// Load does not set the DataAccess or Search if it is already set
if err := sched.Load(sysProvider, newConf, *flagSkipLast, *flagQuiet); err != nil {
slog.Fatal(err)
}
web.ResetSchedule() // Signal web to point to the new DefaultSchedule
go func() {
slog.Infoln("running new schedule")
if !*flagNoChecks {
sched.Run()
}
}()
slog.Infoln("config reload complete")
return nil
}
ruleProvider.SetReload(reload)
go func() {
slog.Fatal(web.Listen(sysProvider.GetHTTPListen(), *flagDev, sysProvider.GetTSDBHost(), reload))
}()
go func() {
if !*flagNoChecks {
sched.Run()
}
}()
go func() {
sc := make(chan os.Signal, 1)
signal.Notify(sc, os.Interrupt, syscall.SIGTERM)
killing := false
for range sc {
if killing {
slog.Infoln("Second interrupt: exiting")
os.Exit(1)
}
killing = true
go func() {
slog.Infoln("Interrupt: closing down...")
sched.Close(false)
slog.Infoln("done")
os.Exit(1)
}()
}
}()
if *flagWatch {
watch(".", "*.go", quit)
watch(filepath.Join("web", "static", "templates"), "*.html", web.RunEsc)
base := filepath.Join("web", "static", "js")
watch(base, "*.ts", web.RunTsc)
}
select {}
}