// close flushes the indexing queue and other buffered data and releases any
// held resources. It also removes the dirty marker file if successful and if
// the persistence is currently not marked as dirty.
func (p *persistence) close() error {
close(p.indexingQueue)
<-p.indexingStopped
var lastError, dirtyFileRemoveError error
if err := p.archivedFingerprintToMetrics.Close(); err != nil {
lastError = err
log.Error("Error closing archivedFingerprintToMetric index DB: ", err)
}
if err := p.archivedFingerprintToTimeRange.Close(); err != nil {
lastError = err
log.Error("Error closing archivedFingerprintToTimeRange index DB: ", err)
}
if err := p.labelPairToFingerprints.Close(); err != nil {
lastError = err
log.Error("Error closing labelPairToFingerprints index DB: ", err)
}
if err := p.labelNameToLabelValues.Close(); err != nil {
lastError = err
log.Error("Error closing labelNameToLabelValues index DB: ", err)
}
if lastError == nil && !p.isDirty() {
dirtyFileRemoveError = os.Remove(p.dirtyFileName)
}
if err := p.fLock.Release(); err != nil {
lastError = err
log.Error("Error releasing file lock: ", err)
}
if dirtyFileRemoveError != nil {
// On Windows, removing the dirty file before unlocking is not
// possible. So remove it here if it failed above.
lastError = os.Remove(p.dirtyFileName)
}
return lastError
}
func (w *fileWatcher) Watch(cb ReloadCallback) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Fatal(err)
}
err = watcher.WatchFlags(w.fileName, fsnotify.FSN_MODIFY)
if err != nil {
log.Fatal(err)
}
for {
select {
case ev := <-watcher.Event:
log.Infof("Config file changed (%s), attempting reload", ev)
conf, err := LoadFromFile(w.fileName)
if err != nil {
log.Error("Error loading new config: ", err)
failedConfigReloads.Inc()
} else {
cb(&conf)
log.Info("Config reloaded successfully")
configReloads.Inc()
}
// Re-add the file watcher since it can get lost on some changes. E.g.
// saving a file with vim results in a RENAME-MODIFY-DELETE event
// sequence, after which the newly written file is no longer watched.
err = watcher.WatchFlags(w.fileName, fsnotify.FSN_MODIFY)
case err := <-watcher.Error:
log.Error("Error watching config: ", err)
}
}
}
// closeChunkFile first syncs the provided file if mandated so by the sync
// strategy. Then it closes the file. Errors are logged.
func (p *persistence) closeChunkFile(f *os.File) {
if p.shouldSync() {
if err := f.Sync(); err != nil {
log.Error("Error syncing file:", err)
}
}
if err := f.Close(); err != nil {
log.Error("Error closing chunk file:", err)
}
}
func (h *Handler) executeTemplate(w http.ResponseWriter, name string, data interface{}) {
tpl, err := h.getTemplate(name)
if err != nil {
log.Error("Error preparing layout template: ", err)
return
}
err = tpl.Execute(w, data)
if err != nil {
log.Error("Error executing template: ", err)
}
}
func executeTemplate(w http.ResponseWriter, name string, data interface{}, pathPrefix string) {
tpl, err := getTemplate(name, pathPrefix)
if err != nil {
log.Error("Error preparing layout template: ", err)
return
}
err = tpl.Execute(w, data)
if err != nil {
log.Error("Error executing template: ", err)
}
}
// persistChunks persists a number of consecutive chunks of a series. It is the
// caller's responsibility to not modify the chunks concurrently and to not
// persist or drop anything for the same fingerprint concurrently. It returns
// the (zero-based) index of the first persisted chunk within the series
// file. In case of an error, the returned index is -1 (to avoid the
// misconception that the chunk was written at position 0).
func (p *persistence) persistChunks(fp clientmodel.Fingerprint, chunks []chunk) (index int, err error) {
defer func() {
if err != nil {
log.Error("Error persisting chunks: ", err)
p.setDirty(true)
}
}()
f, err := p.openChunkFileForWriting(fp)
if err != nil {
return -1, err
}
defer p.closeChunkFile(f)
if err := writeChunks(f, chunks); err != nil {
return -1, err
}
// Determine index within the file.
offset, err := f.Seek(0, os.SEEK_CUR)
if err != nil {
return -1, err
}
index, err = chunkIndexForOffset(offset)
if err != nil {
return -1, err
}
return index - len(chunks), err
}
// GetJson return json from server
func GetJson(url string, accessKey string, secretKey string, target interface{}) error {
start := time.Now()
// Counter for internal exporter metrics
measure.FunctionCountTotal.With(prometheus.Labels{"pkg": "utils", "fnc": "GetJson"}).Inc()
log.Info("Scraping: ", url)
client := &http.Client{}
req, err := http.NewRequest("GET", url, nil)
req.SetBasicAuth(accessKey, secretKey)
resp, err := client.Do(req)
if err != nil {
log.Error("Error Collecting JSON from API: ", err)
panic(err)
}
// Timings recorded as part of internal metrics
elapsed := float64((time.Since(start)) / time.Microsecond)
measure.FunctionDurations.WithLabelValues("hosts", "getJSON").Observe(elapsed)
return json.NewDecoder(resp.Body).Decode(target)
}
func dumpHeap(w http.ResponseWriter, r *http.Request) {
target := fmt.Sprintf("/tmp/%d.heap", time.Now().Unix())
f, err := os.Create(target)
if err != nil {
log.Error("Could not dump heap: ", err)
}
fmt.Fprintf(w, "Writing to %s...", target)
defer f.Close()
pprof_runtime.WriteHeapProfile(f)
fmt.Fprintf(w, "Done")
}
func (h *Handler) getTemplate(name string) (*template_std.Template, error) {
t := template_std.New("_base")
var err error
t.Funcs(template_std.FuncMap{
"since": time.Since,
"getConsoles": h.getConsoles,
"pathPrefix": func() string { return h.options.PathPrefix },
"stripLabels": func(lset clientmodel.LabelSet, labels ...clientmodel.LabelName) clientmodel.LabelSet {
for _, ln := range labels {
delete(lset, ln)
}
return lset
},
"globalURL": func(url string) string {
for _, localhostRepresentation := range localhostRepresentations {
url = strings.Replace(url, "//"+localhostRepresentation, "//"+h.options.Hostname, 1)
}
return url
},
"healthToClass": func(th retrieval.TargetHealth) string {
switch th {
case retrieval.HealthUnknown:
return "warning"
case retrieval.HealthGood:
return "success"
default:
return "danger"
}
},
})
file, err := h.getTemplateFile("_base")
if err != nil {
log.Errorln("Could not read base template:", err)
return nil, err
}
t, err = t.Parse(file)
if err != nil {
log.Errorln("Could not parse base template:", err)
}
file, err = h.getTemplateFile(name)
if err != nil {
log.Error("Could not read template %s: %s", name, err)
return nil, err
}
t, err = t.Parse(file)
if err != nil {
log.Errorf("Could not parse template %s: %s", name, err)
}
return t, err
}
// setDirty sets the dirty flag in a goroutine-safe way. Once the dirty flag was
// set to true with this method, it cannot be set to false again. (If we became
// dirty during our runtime, there is no way back. If we were dirty from the
// start, a clean-up might make us clean again.)
func (p *persistence) setDirty(dirty bool) {
p.dirtyCounter.Inc()
p.dirtyMtx.Lock()
defer p.dirtyMtx.Unlock()
if p.becameDirty {
return
}
p.dirty = dirty
if dirty {
p.becameDirty = true
log.Error("The storage is now inconsistent. Restart Prometheus ASAP to initiate recovery.")
}
}
// maintainArchivedSeries drops chunks older than beforeTime from an archived
// series. If the series contains no chunks after that, it is purged entirely.
func (s *memorySeriesStorage) maintainArchivedSeries(fp model.Fingerprint, beforeTime model.Time) {
defer func(begin time.Time) {
s.maintainSeriesDuration.WithLabelValues(maintainArchived).Observe(
float64(time.Since(begin)) / float64(time.Millisecond),
)
}(time.Now())
s.fpLocker.Lock(fp)
defer s.fpLocker.Unlock(fp)
has, firstTime, lastTime, err := s.persistence.hasArchivedMetric(fp)
if err != nil {
log.Error("Error looking up archived time range: ", err)
return
}
if !has || !firstTime.Before(beforeTime) {
// Oldest sample not old enough, or metric purged or unarchived in the meantime.
return
}
defer s.seriesOps.WithLabelValues(archiveMaintenance).Inc()
newFirstTime, _, _, allDropped, err := s.persistence.dropAndPersistChunks(fp, beforeTime, nil)
if err != nil {
log.Error("Error dropping persisted chunks: ", err)
}
if allDropped {
if err := s.persistence.purgeArchivedMetric(fp); err != nil {
log.Errorf("Error purging archived metric for fingerprint %v: %v", fp, err)
return
}
s.seriesOps.WithLabelValues(archivePurge).Inc()
return
}
if err := s.persistence.updateArchivedTimeRange(fp, newFirstTime, lastTime); err != nil {
log.Errorf("Error updating archived time range for fingerprint %v: %s", fp, err)
}
}
// Metrics handles the /api/metrics endpoint.
func (api *API) Metrics(w http.ResponseWriter, r *http.Request) {
setAccessControlHeaders(w)
w.Header().Set("Content-Type", "application/json")
metricNames := api.Storage.LabelValuesForLabelName(clientmodel.MetricNameLabel)
sort.Sort(metricNames)
resultBytes, err := json.Marshal(metricNames)
if err != nil {
log.Error("Error marshalling metric names: ", err)
httpJSONError(w, fmt.Errorf("Error marshalling metric names: %s", err), http.StatusInternalServerError)
return
}
w.Write(resultBytes)
}
// Run dispatches notifications continuously.
func (n *NotificationHandler) Run() {
for reqs := range n.pendingNotifications {
if n.alertmanagerURL == "" {
log.Warn("No alert manager configured, not dispatching notification")
n.notificationDropped.Inc()
continue
}
begin := time.Now()
err := n.sendNotifications(reqs)
if err != nil {
log.Error("Error sending notification: ", err)
n.notificationErrors.Inc()
}
n.notificationLatency.Observe(float64(time.Since(begin) / time.Millisecond))
}
close(n.stopped)
}
func readTanks() {
var token_response tank_utility.TokenResponse
if *token_file != "" {
token_response = tank_utility.ReadTokenFromFile(*token_file)
} else {
log.Error("Could not get token.")
}
token := token_response.Token
// TODO(kendall): Handle invalid or no token_file set.
device_list := tank_utility.GetDeviceList(token, *tank_utility_endpoint, *insecure).Devices
for i := 0; i < len(device_list); i++ {
var device_info tank_utility.DeviceInfo
device_info = tank_utility.GetDeviceInfo(device_list[i], token, *tank_utility_endpoint, *insecure)
timestamp.WithLabelValues(device_info.Device.Name).Set(float64(device_info.Device.LastReading.Time))
capacity.WithLabelValues(device_info.Device.Name).Set(float64(device_info.Device.Capacity))
percentage.WithLabelValues(device_info.Device.Name).Set(device_info.Device.LastReading.Tank)
temperature.WithLabelValues(device_info.Device.Name).Set(float64(device_info.Device.LastReading.Temperature))
}
}
func (e *Exporter) gatherMetrics(rancherURL string, accessKey string, secretKey string, ch chan<- prometheus.Metric) error {
// Reset guageVecs back to 0
for _, m := range e.gaugeVecs {
m.Reset()
}
// Set the correct API endpoint for hosts
endpoint := (rancherURL + "/hosts/")
// Scrape EndPoint for JSON Data
data := new(Data)
err := utils.GetJson(endpoint, accessKey, secretKey, &data)
if err != nil {
log.Error("Error getting JSON from URL ", endpoint)
return err
}
log.Info("JSON Fetched for hosts: ", data)
// Host Metrics
for _, x := range data.Data {
// Pre-defines the known states from the Rancher API
states := []string{"activating", "active", "deactivating", "error", "erroring", "inactive", "provisioned", "purged", "purging", "registering", "removed", "removing", "requested", "restoring", "updating_active", "updating_inactive"}
// Set the state of the service to 1 when it matches one of the known states
for _, y := range states {
if x.State == y {
e.gaugeVecs["HostState"].With(prometheus.Labels{"rancherURL": rancherURL, "name": x.Hostname, "state": y}).Set(1)
} else {
e.gaugeVecs["HostState"].With(prometheus.Labels{"rancherURL": rancherURL, "name": x.Hostname, "state": y}).Set(0)
}
}
}
return nil
}
func getEmbeddedTemplate(name string, pathPrefix string) (*template.Template, error) {
t := template.New("_base.html")
t.Funcs(webHelpers)
t.Funcs(template.FuncMap{"pathPrefix": func() string { return pathPrefix }})
file, err := blob.GetFile(blob.TemplateFiles, "_base.html")
if err != nil {
log.Error("Could not read base template: ", err)
return nil, err
}
t.Parse(string(file))
file, err = blob.GetFile(blob.TemplateFiles, name+".html")
if err != nil {
log.Errorf("Could not read %s template: %s", name, err)
return nil, err
}
t.Parse(string(file))
return t, nil
}
// cycleThroughArchivedFingerprints returns a channel that emits fingerprints
// for archived series in a throttled fashion. It continues to cycle through all
// archived fingerprints until s.loopStopping is closed.
func (s *memorySeriesStorage) cycleThroughArchivedFingerprints() chan model.Fingerprint {
archivedFingerprints := make(chan model.Fingerprint)
go func() {
defer close(archivedFingerprints)
for {
archivedFPs, err := s.persistence.fingerprintsModifiedBefore(
model.Now().Add(-s.dropAfter),
)
if err != nil {
log.Error("Failed to lookup archived fingerprint ranges: ", err)
s.waitForNextFP(0, 1)
continue
}
// Initial wait, also important if there are no FPs yet.
if !s.waitForNextFP(len(archivedFPs), 1) {
return
}
begin := time.Now()
for _, fp := range archivedFPs {
select {
case archivedFingerprints <- fp:
case <-s.loopStopping:
return
}
// Never speed up maintenance of archived FPs.
s.waitForNextFP(len(archivedFPs), 1)
}
if len(archivedFPs) > 0 {
log.Infof(
"Completed maintenance sweep through %d archived fingerprints in %v.",
len(archivedFPs), time.Since(begin),
)
}
}
}()
return archivedFingerprints
}
// fingerprintsForLabelPairs returns the set of fingerprints that have the given labels.
// This does not work with empty label values.
func (s *memorySeriesStorage) fingerprintsForLabelPairs(pairs ...model.LabelPair) map[model.Fingerprint]struct{} {
var result map[model.Fingerprint]struct{}
for _, pair := range pairs {
intersection := map[model.Fingerprint]struct{}{}
fps, err := s.persistence.fingerprintsForLabelPair(pair)
if err != nil {
log.Error("Error getting fingerprints for label pair: ", err)
}
if len(fps) == 0 {
return nil
}
for _, fp := range fps {
if _, ok := result[fp]; ok || result == nil {
intersection[fp] = struct{}{}
}
}
if len(intersection) == 0 {
return nil
}
result = intersection
}
return result
}
func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
name := r.URL.Path
if name == "" {
name = "index.html"
}
file, err := GetFile(StaticFiles, name)
if err != nil {
if err != io.EOF {
log.Error("Could not get file: ", err)
}
w.WriteHeader(http.StatusNotFound)
return
}
contentType := http.DetectContentType(file)
if strings.Contains(contentType, "text/plain") || strings.Contains(contentType, "application/octet-stream") {
parts := strings.Split(name, ".")
contentType = mimeMap[parts[len(parts)-1]]
}
w.Header().Set("Content-Type", contentType)
w.Header().Set("Cache-Control", "public, max-age=259200")
w.Write(file)
}
// dropAndPersistChunks deletes all chunks from a series file whose last sample
// time is before beforeTime, and then appends the provided chunks, leaving out
// those whose last sample time is before beforeTime. It returns the timestamp
// of the first sample in the oldest chunk _not_ dropped, the offset within the
// series file of the first chunk persisted (out of the provided chunks), the
// number of deleted chunks, and true if all chunks of the series have been
// deleted (in which case the returned timestamp will be 0 and must be ignored).
// It is the caller's responsibility to make sure nothing is persisted or loaded
// for the same fingerprint concurrently.
func (p *persistence) dropAndPersistChunks(
fp clientmodel.Fingerprint, beforeTime clientmodel.Timestamp, chunks []chunk,
) (
firstTimeNotDropped clientmodel.Timestamp,
offset int,
numDropped int,
allDropped bool,
err error,
) {
// Style note: With the many return values, it was decided to use naked
// returns in this method. They make the method more readable, but
// please handle with care!
defer func() {
if err != nil {
log.Error("Error dropping and/or persisting chunks: ", err)
p.setDirty(true)
}
}()
if len(chunks) > 0 {
// We have chunks to persist. First check if those are already
// too old. If that's the case, the chunks in the series file
// are all too old, too.
i := 0
for ; i < len(chunks) && chunks[i].newIterator().lastTimestamp().Before(beforeTime); i++ {
}
if i < len(chunks) {
firstTimeNotDropped = chunks[i].firstTime()
}
if i > 0 || firstTimeNotDropped.Before(beforeTime) {
// Series file has to go.
if numDropped, err = p.deleteSeriesFile(fp); err != nil {
return
}
numDropped += i
if i == len(chunks) {
allDropped = true
return
}
// Now simply persist what has to be persisted to a new file.
_, err = p.persistChunks(fp, chunks[i:])
return
}
}
// If we are here, we have to check the series file itself.
f, err := p.openChunkFileForReading(fp)
if os.IsNotExist(err) {
// No series file. Only need to create new file with chunks to
// persist, if there are any.
if len(chunks) == 0 {
allDropped = true
err = nil // Do not report not-exist err.
return
}
offset, err = p.persistChunks(fp, chunks)
return
}
if err != nil {
return
}
defer f.Close()
// Find the first chunk in the file that should be kept.
for ; ; numDropped++ {
_, err = f.Seek(offsetForChunkIndex(numDropped), os.SEEK_SET)
if err != nil {
return
}
headerBuf := make([]byte, chunkHeaderLen)
_, err = io.ReadFull(f, headerBuf)
if err == io.EOF {
// We ran into the end of the file without finding any chunks that should
// be kept. Remove the whole file.
if numDropped, err = p.deleteSeriesFile(fp); err != nil {
return
}
if len(chunks) == 0 {
allDropped = true
return
}
offset, err = p.persistChunks(fp, chunks)
return
}
if err != nil {
return
}
lastTime := clientmodel.Timestamp(
binary.LittleEndian.Uint64(headerBuf[chunkHeaderLastTimeOffset:]),
)
if !lastTime.Before(beforeTime) {
firstTimeNotDropped = clientmodel.Timestamp(
binary.LittleEndian.Uint64(headerBuf[chunkHeaderFirstTimeOffset:]),
)
chunkOps.WithLabelValues(drop).Add(float64(numDropped))
break
}
}
// We've found the first chunk that should be kept. If it is the first
// one, just append the chunks.
if numDropped == 0 {
if len(chunks) > 0 {
offset, err = p.persistChunks(fp, chunks)
}
return
}
// Otherwise, seek backwards to the beginning of its header and start
// copying everything from there into a new file. Then append the chunks
// to the new file.
_, err = f.Seek(-chunkHeaderLen, os.SEEK_CUR)
if err != nil {
return
}
temp, err := os.OpenFile(p.tempFileNameForFingerprint(fp), os.O_WRONLY|os.O_CREATE, 0640)
if err != nil {
return
}
defer func() {
p.closeChunkFile(temp)
if err == nil {
err = os.Rename(p.tempFileNameForFingerprint(fp), p.fileNameForFingerprint(fp))
}
}()
written, err := io.Copy(temp, f)
if err != nil {
return
}
offset = int(written / chunkLenWithHeader)
if len(chunks) > 0 {
if err = writeChunks(temp, chunks); err != nil {
return
}
}
return
}
// FingerprintsForLabelMatchers implements Storage.
func (s *memorySeriesStorage) FingerprintsForLabelMatchers(labelMatchers metric.LabelMatchers) clientmodel.Fingerprints {
var result map[clientmodel.Fingerprint]struct{}
for _, matcher := range labelMatchers {
intersection := map[clientmodel.Fingerprint]struct{}{}
switch matcher.Type {
case metric.Equal:
fps, err := s.persistence.fingerprintsForLabelPair(
metric.LabelPair{
Name: matcher.Name,
Value: matcher.Value,
},
)
if err != nil {
log.Error("Error getting fingerprints for label pair: ", err)
}
if len(fps) == 0 {
return nil
}
for _, fp := range fps {
if _, ok := result[fp]; ok || result == nil {
intersection[fp] = struct{}{}
}
}
default:
values, err := s.persistence.labelValuesForLabelName(matcher.Name)
if err != nil {
log.Errorf("Error getting label values for label name %q: %v", matcher.Name, err)
}
matches := matcher.Filter(values)
if len(matches) == 0 {
return nil
}
for _, v := range matches {
fps, err := s.persistence.fingerprintsForLabelPair(
metric.LabelPair{
Name: matcher.Name,
Value: v,
},
)
if err != nil {
log.Error("Error getting fingerprints for label pair: ", err)
}
for _, fp := range fps {
if _, ok := result[fp]; ok || result == nil {
intersection[fp] = struct{}{}
}
}
}
}
if len(intersection) == 0 {
return nil
}
result = intersection
}
fps := make(clientmodel.Fingerprints, 0, len(result))
for fp := range result {
fps = append(fps, fp)
}
return fps
}
func (p *persistence) processIndexingQueue() {
batchSize := 0
nameToValues := index.LabelNameLabelValuesMapping{}
pairToFPs := index.LabelPairFingerprintsMapping{}
batchTimeout := time.NewTimer(indexingBatchTimeout)
defer batchTimeout.Stop()
commitBatch := func() {
p.indexingBatchSizes.Observe(float64(batchSize))
defer func(begin time.Time) {
p.indexingBatchDuration.Observe(
float64(time.Since(begin)) / float64(time.Millisecond),
)
}(time.Now())
if err := p.labelPairToFingerprints.IndexBatch(pairToFPs); err != nil {
log.Error("Error indexing label pair to fingerprints batch: ", err)
}
if err := p.labelNameToLabelValues.IndexBatch(nameToValues); err != nil {
log.Error("Error indexing label name to label values batch: ", err)
}
batchSize = 0
nameToValues = index.LabelNameLabelValuesMapping{}
pairToFPs = index.LabelPairFingerprintsMapping{}
batchTimeout.Reset(indexingBatchTimeout)
}
var flush chan chan int
loop:
for {
// Only process flush requests if the queue is currently empty.
if len(p.indexingQueue) == 0 {
flush = p.indexingFlush
} else {
flush = nil
}
select {
case <-batchTimeout.C:
// Only commit if we have something to commit _and_
// nothing is waiting in the queue to be picked up. That
// prevents a death spiral if the LookupSet calls below
// are slow for some reason.
if batchSize > 0 && len(p.indexingQueue) == 0 {
commitBatch()
} else {
batchTimeout.Reset(indexingBatchTimeout)
}
case r := <-flush:
if batchSize > 0 {
commitBatch()
}
r <- len(p.indexingQueue)
case op, ok := <-p.indexingQueue:
if !ok {
if batchSize > 0 {
commitBatch()
}
break loop
}
batchSize++
for ln, lv := range op.metric {
lp := metric.LabelPair{Name: ln, Value: lv}
baseFPs, ok := pairToFPs[lp]
if !ok {
var err error
baseFPs, _, err = p.labelPairToFingerprints.LookupSet(lp)
if err != nil {
log.Errorf("Error looking up label pair %v: %s", lp, err)
continue
}
pairToFPs[lp] = baseFPs
}
baseValues, ok := nameToValues[ln]
if !ok {
var err error
baseValues, _, err = p.labelNameToLabelValues.LookupSet(ln)
if err != nil {
log.Errorf("Error looking up label name %v: %s", ln, err)
continue
}
nameToValues[ln] = baseValues
}
switch op.opType {
case add:
baseFPs[op.fingerprint] = struct{}{}
baseValues[lv] = struct{}{}
case remove:
delete(baseFPs, op.fingerprint)
if len(baseFPs) == 0 {
delete(baseValues, lv)
}
default:
panic("unknown op type")
}
}
if batchSize >= indexingMaxBatchSize {
commitBatch()
}
}
}
close(p.indexingStopped)
}
func (e *exporter) recordErr(err error) {
log.Error("Error: ", err)
e.errors.Inc()
}
func main() {
flag.Parse()
if !strings.HasPrefix(*pathPrefix, "/") {
*pathPrefix = "/" + *pathPrefix
}
if !strings.HasSuffix(*pathPrefix, "/") {
*pathPrefix = *pathPrefix + "/"
}
versionInfoTmpl.Execute(os.Stdout, BuildInfo)
conf := config.MustLoadFromFile(*configFile)
silencer := manager.NewSilencer()
defer silencer.Close()
err := silencer.LoadFromFile(*silencesFile)
if err != nil {
log.Warn("Couldn't load silences, starting up with empty silence list: ", err)
}
saveSilencesTicker := time.NewTicker(10 * time.Second)
go func() {
for range saveSilencesTicker.C {
if err := silencer.SaveToFile(*silencesFile); err != nil {
log.Error("Error saving silences to file: ", err)
}
}
}()
defer saveSilencesTicker.Stop()
amURL, err := alertmanagerURL(*hostname, *pathPrefix, *listenAddress, *externalURL)
if err != nil {
log.Fatalln("Error building Alertmanager URL:", err)
}
notifier := manager.NewNotifier(conf.NotificationConfig, amURL)
defer notifier.Close()
inhibitor := new(manager.Inhibitor)
inhibitor.SetInhibitRules(conf.InhibitRules())
options := &manager.MemoryAlertManagerOptions{
Inhibitor: inhibitor,
Silencer: silencer,
Notifier: notifier,
MinRefreshInterval: *minRefreshPeriod,
}
alertManager := manager.NewMemoryAlertManager(options)
alertManager.SetAggregationRules(conf.AggregationRules())
go alertManager.Run()
// Web initialization.
flags := map[string]string{}
flag.VisitAll(func(f *flag.Flag) {
flags[f.Name] = f.Value.String()
})
statusHandler := &web.StatusHandler{
Config: conf.String(),
Flags: flags,
BuildInfo: BuildInfo,
Birth: time.Now(),
PathPrefix: *pathPrefix,
}
webService := &web.WebService{
// REST API Service.
AlertManagerService: &api.AlertManagerService{
Manager: alertManager,
Silencer: silencer,
PathPrefix: *pathPrefix,
},
// Template-based page handlers.
AlertsHandler: &web.AlertsHandler{
Manager: alertManager,
IsSilencedInterrogator: silencer,
PathPrefix: *pathPrefix,
},
SilencesHandler: &web.SilencesHandler{
Silencer: silencer,
PathPrefix: *pathPrefix,
},
StatusHandler: statusHandler,
}
go webService.ServeForever(*listenAddress, *pathPrefix)
// React to configuration changes.
watcher := config.NewFileWatcher(*configFile)
go watcher.Watch(func(conf *config.Config) {
inhibitor.SetInhibitRules(conf.InhibitRules())
notifier.SetNotificationConfigs(conf.NotificationConfig)
alertManager.SetAggregationRules(conf.AggregationRules())
statusHandler.UpdateConfig(conf.String())
})
log.Info("Running notification dispatcher...")
notifier.Dispatch()
}