func DatabaseRunQueryWorker(c conf.Database, db *sql.DB, queryChan <-chan *DatabaseQuery, collectorStatsChan chan<- *ContinuousCollectorStats) {
var md opentsdb.MultiDataPoint
for query := range queryChan {
result := 0
timeStart := time.Now()
results, err := DatabaseRunQuery(db, query.query)
if err == nil {
md, err = DatabaseParseResults(c, query, results)
}
if err != nil {
slog.Warningf("%v: %v: %v: %v: %v: %v", "Database", c.Type, "query error", err, "for query", query.query)
result = 1
}
if !collect.DisableDefaultCollectors {
collectorStatsChan <- &ContinuousCollectorStats{duration: time.Since(timeStart).Seconds(), result: result}
}
for _, dp := range md {
ContinuousCollectorVars.dpChan <- dp
}
md = nil
}
}
//Download a GZipped item from S3 and return the non-gzipped version of the item
func (billingObject *bucketContents) downloadGzippedItem(downloader *s3manager.Downloader, awsBilling *awsBillingConfig) (*[]byte, error) {
//Get a temporary file to dump this into
tempFile, err := ioutil.TempFile("", "scollector-aws-billing-")
if err != nil {
return nil, err
}
defer tempFile.Close()
if _, err = downloader.Download(tempFile,
&s3.GetObjectInput{
Bucket: aws.String(awsBilling.bucketName),
Key: billingObject.origS3Item.Key,
}); err != nil {
return nil, err
}
unzippedFile, err := readGzFile(tempFile)
if err != nil {
return nil, err
}
tempFile.Close()
err = os.Remove(tempFile.Name())
if err != nil {
slog.Warningf("Could not remove temporary file", tempFile.Name())
}
return &unzippedFile, err
}
func startCadvisorCollector(c *conf.Conf) {
for _, config := range c.Cadvisor {
cClient, err := client.NewClient(config.URL)
if err != nil {
slog.Warningf("Could not start collector for URL [%s] due to err: %v", config.URL, err)
}
collectors = append(collectors, &IntervalCollector{
F: func() (opentsdb.MultiDataPoint, error) {
return c_cadvisor(cClient, &config)
},
name: "cadvisor",
})
}
}
func DatabaseGetQueries(c conf.Database) []DatabaseQuery {
queries := make([]DatabaseQuery, 0, len(c.Query))
for _, query := range c.Query {
cleaned, _ := opentsdb.Clean(query.Name)
if query.Name != cleaned {
slog.Warningf("%v: %v: %v: %v", "Database", c.Type, "invaid metric name", query.Name)
continue
}
databaseQuery := DatabaseQuery{name: query.Name, query: query.Query, description: query.Description, hasTime: query.HasTime}
if query.Interval < 1 {
databaseQuery.interval = DefaultFreq
} else {
databaseQuery.interval = time.Duration(query.Interval) * time.Second
}
queries = append(queries, databaseQuery)
}
return queries
}
func Listen(listenAddr string, devMode bool, tsdbHost string, reloadFunc func() error) error {
if devMode {
slog.Infoln("using local web assets")
}
webFS := FS(devMode)
indexTemplate = func() *template.Template {
str := FSMustString(devMode, "/templates/index.html")
templates, err := template.New("").Parse(str)
if err != nil {
slog.Fatal(err)
}
return templates
}
reload = reloadFunc
if !devMode {
tpl := indexTemplate()
indexTemplate = func() *template.Template {
return tpl
}
}
if tsdbHost != "" {
router.HandleFunc("/api/index", IndexTSDB)
router.Handle("/api/put", Relay(tsdbHost))
}
router.HandleFunc("/api/", APIRedirect)
router.Handle("/api/action", JSON(Action))
router.Handle("/api/alerts", JSON(Alerts))
router.Handle("/api/config", miniprofiler.NewHandler(Config))
router.Handle("/api/config_test", miniprofiler.NewHandler(ConfigTest))
router.Handle("/api/save_enabled", JSON(SaveEnabled))
if schedule.SystemConf.ReloadEnabled() { // Is true of save is enabled
router.Handle("/api/reload", JSON(Reload)).Methods(http.MethodPost)
}
if schedule.SystemConf.SaveEnabled() {
router.Handle("/api/config/bulkedit", miniprofiler.NewHandler(BulkEdit)).Methods(http.MethodPost)
router.Handle("/api/config/save", miniprofiler.NewHandler(SaveConfig)).Methods(http.MethodPost)
router.Handle("/api/config/diff", miniprofiler.NewHandler(DiffConfig)).Methods(http.MethodPost)
router.Handle("/api/config/running_hash", JSON(ConfigRunningHash))
}
router.Handle("/api/egraph/{bs}.{format:svg|png}", JSON(ExprGraph))
router.Handle("/api/errors", JSON(ErrorHistory))
router.Handle("/api/expr", JSON(Expr))
router.Handle("/api/graph", JSON(Graph))
router.Handle("/api/health", JSON(HealthCheck))
router.Handle("/api/host", JSON(Host))
router.Handle("/api/last", JSON(Last))
router.Handle("/api/quiet", JSON(Quiet))
router.Handle("/api/incidents", JSON(Incidents))
router.Handle("/api/incidents/open", JSON(ListOpenIncidents))
router.Handle("/api/incidents/events", JSON(IncidentEvents))
router.Handle("/api/metadata/get", JSON(GetMetadata))
router.Handle("/api/metadata/metrics", JSON(MetadataMetrics))
router.Handle("/api/metadata/put", JSON(PutMetadata))
router.Handle("/api/metadata/delete", JSON(DeleteMetadata)).Methods("DELETE")
router.Handle("/api/metric", JSON(UniqueMetrics))
router.Handle("/api/metric/{tagk}", JSON(MetricsByTagKey))
router.Handle("/api/metric/{tagk}/{tagv}", JSON(MetricsByTagPair))
router.Handle("/api/rule", JSON(Rule))
router.HandleFunc("/api/shorten", Shorten)
router.Handle("/api/silence/clear", JSON(SilenceClear))
router.Handle("/api/silence/get", JSON(SilenceGet))
router.Handle("/api/silence/set", JSON(SilenceSet))
router.Handle("/api/status", JSON(Status))
router.Handle("/api/tagk/{metric}", JSON(TagKeysByMetric))
router.Handle("/api/tagv/{tagk}", JSON(TagValuesByTagKey))
router.Handle("/api/tagv/{tagk}/{metric}", JSON(TagValuesByMetricTagKey))
router.Handle("/api/tagsets/{metric}", JSON(FilteredTagsetsByMetric))
router.Handle("/api/opentsdb/version", JSON(OpenTSDBVersion))
router.Handle("/api/annotate", JSON(AnnotateEnabled))
// Annotations
if schedule.SystemConf.AnnotateEnabled() {
index := schedule.SystemConf.GetAnnotateIndex()
if index == "" {
index = "annotate"
}
annotateBackend = backend.NewElastic(schedule.SystemConf.GetAnnotateElasticHosts(), index)
go func() {
for {
err := annotateBackend.InitBackend()
if err == nil {
return
}
slog.Warningf("could not initalize annotate backend, will try again: %v", err)
time.Sleep(time.Second * 30)
}
}()
web.AddRoutes(router, "/api", []backend.Backend{annotateBackend}, false, false)
}
router.HandleFunc("/api/version", Version)
router.Handle("/api/debug/schedlock", JSON(ScheduleLockStatus))
http.Handle("/", miniprofiler.NewHandler(Index))
http.Handle("/api/", router)
fs := http.FileServer(webFS)
http.Handle("/partials/", fs)
http.Handle("/static/", http.StripPrefix("/static/", fs))
http.Handle("/favicon.ico", fs)
slog.Infoln("bosun web listening on:", listenAddr)
slog.Infoln("tsdb host:", tsdbHost)
return http.ListenAndServe(listenAddr, nil)
}
