func TestSilenceStage(t *testing.T) {
silences, err := silence.New(silence.Options{})
if err != nil {
t.Fatal(err)
}
if _, err := silences.Create(&silencepb.Silence{
EndsAt: mustTimestampProto(utcNow().Add(time.Hour)),
Matchers: []*silencepb.Matcher{{Name: "mute", Pattern: "me"}},
}); err != nil {
t.Fatal(err)
}
marker := types.NewMarker()
silencer := NewSilenceStage(silences, marker)
in := []model.LabelSet{
{},
{"test": "set"},
{"mute": "me"},
{"foo": "bar", "test": "set"},
{"foo": "bar", "mute": "me"},
{},
{"not": "muted"},
}
out := []model.LabelSet{
{},
{"test": "set"},
{"foo": "bar", "test": "set"},
{},
{"not": "muted"},
}
var inAlerts []*types.Alert
for _, lset := range in {
inAlerts = append(inAlerts, &types.Alert{
Alert: model.Alert{Labels: lset},
})
}
// Set the second alert als previously silenced. It is expected to have
// the WasSilenced flag set to true afterwards.
marker.SetSilenced(inAlerts[1].Fingerprint(), "123")
_, alerts, err := silencer.Exec(nil, inAlerts...)
if err != nil {
t.Fatalf("Exec failed: %s", err)
}
var got []model.LabelSet
for i, a := range alerts {
got = append(got, a.Labels)
if a.WasSilenced != (i == 1) {
t.Errorf("Expected WasSilenced to be %v for %d, was %v", i == 1, i, a.WasSilenced)
}
}
if !reflect.DeepEqual(got, out) {
t.Fatalf("Muting failed, expected: %v\ngot %v", out, got)
}
}
func TestInhibitNotifier(t *testing.T) {
// Mute all label sets that have a "mute" key.
muter := types.MuteFunc(func(lset model.LabelSet) bool {
_, ok := lset["mute"]
return ok
})
marker := types.NewMarker()
record := &recordNotifier{}
inhibitNotifer := Inhibit(muter, record, marker)
in := []model.LabelSet{
{},
{"test": "set"},
{"mute": "me"},
{"foo": "bar", "test": "set"},
{"foo": "bar", "mute": "me"},
{},
{"not": "muted"},
}
out := []model.LabelSet{
{},
{"test": "set"},
{"foo": "bar", "test": "set"},
{},
{"not": "muted"},
}
var inAlerts []*types.Alert
for _, lset := range in {
inAlerts = append(inAlerts, &types.Alert{
Alert: model.Alert{Labels: lset},
})
}
// Set the second alert as previously inhibited. It is expected to have
// the WasInhibited flag set to true afterwards.
marker.SetInhibited(inAlerts[1].Fingerprint(), true)
if err := inhibitNotifer.Notify(nil, inAlerts...); err != nil {
t.Fatalf("Notifying failed: %s", err)
}
var got []model.LabelSet
for i, a := range record.alerts {
got = append(got, a.Labels)
if a.WasInhibited != (i == 1) {
t.Errorf("Expected WasInhibited to be %v for %d, was %v", i == 1, i, a.WasInhibited)
}
}
if !reflect.DeepEqual(got, out) {
t.Fatalf("Muting failed, expected: %v\ngot %v", out, got)
}
}
func main() {
flag.Parse()
if *showVersion {
fmt.Fprintln(os.Stdout, version.Print("alertmanager"))
os.Exit(0)
}
log.Infoln("Starting alertmanager", version.Info())
log.Infoln("Build context", version.BuildContext())
err := os.MkdirAll(*dataDir, 0777)
if err != nil {
log.Fatal(err)
}
marker := types.NewMarker()
alerts, err := boltmem.NewAlerts(*dataDir)
if err != nil {
log.Fatal(err)
}
defer alerts.Close()
notifies, err := boltmem.NewNotificationInfo(*dataDir)
if err != nil {
log.Fatal(err)
}
defer notifies.Close()
silences, err := boltmem.NewSilences(*dataDir, marker)
if err != nil {
log.Fatal(err)
}
defer silences.Close()
var (
inhibitor *Inhibitor
tmpl *template.Template
disp *Dispatcher
)
defer disp.Stop()
api := NewAPI(alerts, silences, func() AlertOverview {
return disp.Groups()
})
build := func(rcvs []*config.Receiver) notify.Notifier {
var (
router = notify.Router{}
fanouts = notify.Build(rcvs, tmpl)
)
for name, fo := range fanouts {
for i, n := range fo {
n = notify.Retry(n)
n = notify.Log(n, log.With("step", "retry"))
n = notify.Dedup(notifies, n)
n = notify.Log(n, log.With("step", "dedup"))
fo[i] = n
}
router[name] = fo
}
n := notify.Notifier(router)
n = notify.Log(n, log.With("step", "route"))
n = notify.Silence(silences, n, marker)
n = notify.Log(n, log.With("step", "silence"))
n = notify.Inhibit(inhibitor, n, marker)
n = notify.Log(n, log.With("step", "inhibit"))
return n
}
amURL, err := extURL(*externalURL)
if err != nil {
log.Fatal(err)
}
reload := func() (err error) {
log.With("file", *configFile).Infof("Loading configuration file")
defer func() {
if err != nil {
log.With("file", *configFile).Errorf("Loading configuration file failed: %s", err)
configSuccess.Set(0)
} else {
configSuccess.Set(1)
configSuccessTime.Set(float64(time.Now().Unix()))
}
}()
conf, err := config.LoadFile(*configFile)
if err != nil {
return err
}
api.Update(conf.String(), time.Duration(conf.Global.ResolveTimeout))
tmpl, err = template.FromGlobs(conf.Templates...)
if err != nil {
return err
}
tmpl.ExternalURL = amURL
inhibitor.Stop()
disp.Stop()
inhibitor = NewInhibitor(alerts, conf.InhibitRules, marker)
disp = NewDispatcher(alerts, NewRoute(conf.Route, nil), build(conf.Receivers), marker)
go disp.Run()
go inhibitor.Run()
return nil
}
if err := reload(); err != nil {
os.Exit(1)
}
router := route.New()
webReload := make(chan struct{})
RegisterWeb(router.WithPrefix(amURL.Path), webReload)
api.Register(router.WithPrefix(path.Join(amURL.Path, "/api")))
log.Infoln("Listening on", *listenAddress)
go listen(router)
var (
hup = make(chan os.Signal)
hupReady = make(chan bool)
term = make(chan os.Signal)
)
signal.Notify(hup, syscall.SIGHUP)
signal.Notify(term, os.Interrupt, syscall.SIGTERM)
go func() {
<-hupReady
for {
select {
case <-hup:
case <-webReload:
}
reload()
}
}()
// Wait for reload or termination signals.
close(hupReady) // Unblock SIGHUP handler.
<-term
log.Infoln("Received SIGTERM, exiting gracefully...")
}
func main() {
flag.Parse()
printVersion()
if *showVersion {
os.Exit(0)
}
err := os.MkdirAll(*dataDir, 0777)
if err != nil {
log.Fatal(err)
}
db, err := sql.Open("sqlite3", filepath.Join(*dataDir, "am.db"))
if err != nil {
log.Fatal(err)
}
defer db.Close()
marker := types.NewMarker()
alerts, err := sqlite.NewAlerts(db)
if err != nil {
log.Fatal(err)
}
notifies, err := sqlite.NewNotifies(db)
if err != nil {
log.Fatal(err)
}
silences, err := sqlite.NewSilences(db, marker)
if err != nil {
log.Fatal(err)
}
var (
inhibitor *Inhibitor
tmpl *template.Template
disp *Dispatcher
)
defer disp.Stop()
api := NewAPI(alerts, silences, func() AlertOverview {
return disp.Groups()
})
build := func(rcvs []*config.Receiver) notify.Notifier {
var (
router = notify.Router{}
fanouts = notify.Build(rcvs, tmpl)
)
for name, fo := range fanouts {
for i, n := range fo {
n = notify.Retry(n)
n = notify.Log(n, log.With("step", "retry"))
n = notify.Dedup(notifies, n)
n = notify.Log(n, log.With("step", "dedup"))
fo[i] = n
}
router[name] = fo
}
n := notify.Notifier(router)
n = notify.Log(n, log.With("step", "route"))
n = notify.Silence(silences, n, marker)
n = notify.Log(n, log.With("step", "silence"))
n = notify.Inhibit(inhibitor, n, marker)
n = notify.Log(n, log.With("step", "inhibit"))
return n
}
reload := func() (err error) {
log.With("file", *configFile).Infof("Loading configuration file")
defer func() {
if err != nil {
log.With("file", *configFile).Errorf("Loading configuration file failed: %s", err)
}
}()
conf, err := config.LoadFile(*configFile)
if err != nil {
return err
}
api.Update(conf.String(), time.Duration(conf.Global.ResolveTimeout))
tmpl, err = template.FromGlobs(conf.Templates...)
if err != nil {
return err
}
if tmpl.ExternalURL, err = extURL(*externalURL); err != nil {
return err
}
disp.Stop()
inhibitor = NewInhibitor(alerts, conf.InhibitRules, marker)
disp = NewDispatcher(alerts, NewRoute(conf.Route, nil), build(conf.Receivers), marker)
go disp.Run()
return nil
}
if err := reload(); err != nil {
os.Exit(1)
}
router := route.New()
RegisterWeb(router)
api.Register(router.WithPrefix("/api"))
go http.ListenAndServe(*listenAddress, router)
var (
hup = make(chan os.Signal)
term = make(chan os.Signal)
)
signal.Notify(hup, syscall.SIGHUP)
signal.Notify(term, os.Interrupt, syscall.SIGTERM)
go func() {
for range hup {
reload()
}
}()
<-term
log.Infoln("Received SIGTERM, exiting gracefully...")
}
func main() {
peers := &stringset{}
var (
showVersion = flag.Bool("version", false, "Print version information.")
configFile = flag.String("config.file", "alertmanager.yml", "Alertmanager configuration file name.")
dataDir = flag.String("storage.path", "data/", "Base path for data storage.")
retention = flag.Duration("data.retention", 5*24*time.Hour, "How long to keep data for.")
externalURL = flag.String("web.external-url", "", "The URL under which Alertmanager is externally reachable (for example, if Alertmanager is served via a reverse proxy). Used for generating relative and absolute links back to Alertmanager itself. If the URL has a path portion, it will be used to prefix all HTTP endpoints served by Alertmanager. If omitted, relevant URL components will be derived automatically.")
listenAddress = flag.String("web.listen-address", ":9093", "Address to listen on for the web interface and API.")
meshListen = flag.String("mesh.listen-address", net.JoinHostPort("0.0.0.0", strconv.Itoa(mesh.Port)), "mesh listen address")
hwaddr = flag.String("mesh.hardware-address", mustHardwareAddr(), "MAC address, i.e. mesh peer ID")
nickname = flag.String("mesh.nickname", mustHostname(), "peer nickname")
)
flag.Var(peers, "mesh.peer", "initial peers (may be repeated)")
flag.Parse()
if len(flag.Args()) > 0 {
log.Fatalln("Received unexpected and unparsed arguments: ", strings.Join(flag.Args(), ", "))
}
if *showVersion {
fmt.Fprintln(os.Stdout, version.Print("alertmanager"))
os.Exit(0)
}
log.Infoln("Starting alertmanager", version.Info())
log.Infoln("Build context", version.BuildContext())
err := os.MkdirAll(*dataDir, 0777)
if err != nil {
log.Fatal(err)
}
logger := log.NewLogger(os.Stderr)
mrouter := initMesh(*meshListen, *hwaddr, *nickname)
stopc := make(chan struct{})
var wg sync.WaitGroup
wg.Add(1)
notificationLog, err := nflog.New(
nflog.WithMesh(func(g mesh.Gossiper) mesh.Gossip {
return mrouter.NewGossip("nflog", g)
}),
nflog.WithRetention(*retention),
nflog.WithSnapshot(filepath.Join(*dataDir, "nflog")),
nflog.WithMaintenance(15*time.Minute, stopc, wg.Done),
nflog.WithLogger(logger.With("component", "nflog")),
)
if err != nil {
log.Fatal(err)
}
marker := types.NewMarker()
silences, err := silence.New(silence.Options{
SnapshotFile: filepath.Join(*dataDir, "silences"),
Retention: *retention,
Logger: logger.With("component", "silences"),
Gossip: func(g mesh.Gossiper) mesh.Gossip {
return mrouter.NewGossip("silences", g)
},
})
if err != nil {
log.Fatal(err)
}
// Start providers before router potentially sends updates.
wg.Add(1)
go func() {
silences.Maintenance(15*time.Minute, filepath.Join(*dataDir, "silences"), stopc)
wg.Done()
}()
mrouter.Start()
defer func() {
close(stopc)
// Stop receiving updates from router before shutting down.
mrouter.Stop()
wg.Wait()
}()
mrouter.ConnectionMaker.InitiateConnections(peers.slice(), true)
alerts, err := mem.NewAlerts(*dataDir)
if err != nil {
log.Fatal(err)
}
defer alerts.Close()
var (
inhibitor *inhibit.Inhibitor
tmpl *template.Template
pipeline notify.Stage
disp *dispatch.Dispatcher
)
defer disp.Stop()
apiv := api.New(alerts, silences, func() dispatch.AlertOverview {
return disp.Groups()
})
amURL, err := extURL(*listenAddress, *externalURL)
if err != nil {
log.Fatal(err)
}
waitFunc := meshWait(mrouter, 5*time.Second)
timeoutFunc := func(d time.Duration) time.Duration {
if d < notify.MinTimeout {
d = notify.MinTimeout
}
return d + waitFunc()
}
reload := func() (err error) {
log.With("file", *configFile).Infof("Loading configuration file")
defer func() {
if err != nil {
log.With("file", *configFile).Errorf("Loading configuration file failed: %s", err)
configSuccess.Set(0)
} else {
configSuccess.Set(1)
configSuccessTime.Set(float64(time.Now().Unix()))
}
}()
conf, err := config.LoadFile(*configFile)
if err != nil {
return err
}
apiv.Update(conf.String(), time.Duration(conf.Global.ResolveTimeout))
tmpl, err = template.FromGlobs(conf.Templates...)
if err != nil {
return err
}
tmpl.ExternalURL = amURL
inhibitor.Stop()
disp.Stop()
inhibitor = inhibit.NewInhibitor(alerts, conf.InhibitRules, marker)
pipeline = notify.BuildPipeline(
conf.Receivers,
tmpl,
waitFunc,
inhibitor,
silences,
notificationLog,
marker,
)
disp = dispatch.NewDispatcher(alerts, dispatch.NewRoute(conf.Route, nil), pipeline, marker, timeoutFunc)
go disp.Run()
go inhibitor.Run()
return nil
}
if err := reload(); err != nil {
os.Exit(1)
}
router := route.New()
webReload := make(chan struct{})
ui.Register(router.WithPrefix(amURL.Path), webReload)
apiv.Register(router.WithPrefix(path.Join(amURL.Path, "/api")))
log.Infoln("Listening on", *listenAddress)
go listen(*listenAddress, router)
var (
hup = make(chan os.Signal)
hupReady = make(chan bool)
term = make(chan os.Signal)
)
signal.Notify(hup, syscall.SIGHUP)
signal.Notify(term, os.Interrupt, syscall.SIGTERM)
go func() {
<-hupReady
for {
select {
case <-hup:
case <-webReload:
}
reload()
}
}()
// Wait for reload or termination signals.
close(hupReady) // Unblock SIGHUP handler.
<-term
log.Infoln("Received SIGTERM, exiting gracefully...")
}
func TestSilencesMutes(t *testing.T) {
var (
t0 = time.Now()
t1 = t0.Add(10 * time.Minute)
t2 = t0.Add(20 * time.Minute)
t3 = t0.Add(30 * time.Minute)
)
// All silences are active for the time of the test. Time restriction
// testing is covered for the Mutes() method of the Silence type.
insert := []*types.Silence{
types.NewSilence(&model.Silence{
Matchers: []*model.Matcher{
{Name: "key", Value: "val"},
},
StartsAt: t0,
EndsAt: t2,
CreatedAt: t1,
CreatedBy: "user",
Comment: "test comment",
}),
types.NewSilence(&model.Silence{
Matchers: []*model.Matcher{
{Name: "key2", Value: "val2.*", IsRegex: true},
},
StartsAt: t0,
EndsAt: t2,
CreatedAt: t1,
CreatedBy: "user2",
Comment: "test comment",
}),
types.NewSilence(&model.Silence{
Matchers: []*model.Matcher{
{Name: "key", Value: "val2"},
},
StartsAt: t0,
EndsAt: t3,
CreatedAt: t3,
CreatedBy: "user",
Comment: "another test comment",
}),
}
dir, err := ioutil.TempDir("", "silences_test")
if err != nil {
t.Fatal(err)
}
silences, err := NewSilences(dir, types.NewMarker())
if err != nil {
t.Fatal(err)
}
for _, sil := range insert {
uid, err := silences.Set(sil)
if err != nil {
t.Fatalf("Insert failed: %s", err)
}
sil.ID = uid
}
tests := []struct {
lset model.LabelSet
match bool
}{
{
lset: model.LabelSet{
"foo": "bar",
"bar": "foo",
},
match: false,
},
{
lset: model.LabelSet{
"key": "val",
"bar": "foo",
},
match: true,
},
{
lset: model.LabelSet{
"foo": "bar",
"key": "val2",
},
match: true,
},
{
lset: model.LabelSet{
"key2": "bar",
"bar": ":$foo",
},
match: false,
},
{
lset: model.LabelSet{
"key2": "val2",
"bar": "foo",
},
match: true,
},
{
lset: model.LabelSet{
"key2": "val2 foo",
"bar": "foo",
},
match: true,
},
}
for i, test := range tests {
if b := silences.Mutes(test.lset); b != test.match {
t.Errorf("Unexpected mute result: %d", i)
t.Fatalf("Expected %v, got %v", test.match, b)
} else {
if _, wasSilenced := silences.mk.Silenced(test.lset.Fingerprint()); wasSilenced != b {
t.Fatalf("Marker was not set correctly: %d", i)
}
}
}
}