// regenHostFiles utility function that calls regen methods for files/persistence that contain only
// host data. We pass along up/down and in/out service info too -- that should be handled with a different
// method. Currently limited so that we don't write More than fileRewriteInterval seconds.
func regenHosts() {
if limiterOn { // we're already waiting on a file rewrite
logr.LogLine(logr.Linfo, ltagsrc, "limiter already on")
return
}
// do some date math here -- have we waited long enough to write our file?
// now < lastfilewrite + fileRewriteInterval
if time.Now().Before(lastFileWrite.Add(time.Duration(fileRewriteInterval) * time.Second)) {
logr.LogLine(logr.Linfo, ltagsrc, "limiter kicked in")
limiterOn = true
// these statements cause us to wait fileRewriteInterval seconds before continuing
limiter := time.Tick(time.Duration(fileRewriteInterval) * time.Second)
<-limiter
}
// flip back our counters
limiterOn = false
lastFileWrite = time.Now()
logr.LogLine(logr.Linfo, ltagsrc, "generating files")
// do the work
etcdWatcher.BuildMap()
hostMap := etcdWatcher.Map()
nagios.GenerateFiles(hostMap, nagios_host_file, nagios_group_file)
writeHostMap(hostMap)
}
func fixHostKey(hostName string) {
hostName = strings.Replace(hostName[1:], "/", "-", -1)
logr.LogLine(logr.Linfo, ltagsrc, fmt.Sprintf("fixhostkey running: %s %s %s", sshkey_clean_path, sshkey_clean_user, hostName))
_, err := exec.Command(sshkey_clean_path, sshkey_clean_user, hostName).Output()
if err != nil {
logr.LogLine(logr.Lerror, ltagsrc, fmt.Sprintf("key_clean for host %s failed", hostName))
}
}
func execCmd(cmdName string, cmdArgs []string) error {
cmd := exec.Command(cmdName, cmdArgs...)
if err := cmd.Start(); err != nil {
logr.LogLine(logr.Lerror, ltagsrc, fmt.Sprintf("cmd.Start:%s -- %s", cmdName, err))
return err
}
// check for non-zero exit code
if err := cmd.Wait(); err != nil {
logr.LogLine(logr.Lerror, ltagsrc, fmt.Sprintf("cmd.Wait:%s -- %s", cmdName, err))
return err
}
return nil
}
func StartWebService(listenPort string) {
http.HandleFunc("/getall", getAll)
//http.HandleFunc("/loadHosts", loadHosts)
http.HandleFunc("/", services)
logr.LogLine(logr.Linfo, ltagsrc, fmt.Sprintf("Starting webservice on port: %s", listenPort))
http.ListenAndServe(fmt.Sprintf(":%s", listenPort), nil)
}
func extractGroup(s string) string {
slist := strings.Split(s, "-")
if len(slist) != 3 {
logr.LogLine(logr.Linfo, ltagsrc, fmt.Sprintf("Invalid format: %s", s))
}
return slist[1]
}
func execCmdOutput(cmdName string, cmdArgs []string) (string, error) {
cmdOut, err := exec.Command(cmdName, cmdArgs...).Output()
if err != nil {
logr.LogLine(logr.Lerror, ltagsrc, fmt.Sprintf("cmd.exec:%s -- %s", cmdName, err))
return "", err
}
return strings.TrimSpace(string(cmdOut)), nil
}
// ClientGet gets data from etcd sending in an url and receiving a etcd.Response object
func ClientGet(kapi client.KeysAPI, url string) *client.Response {
resp, err := kapi.Get(context.Background(), url, &clientGetOpts)
if err != nil {
logr.LogLine(logr.Lfatal, ltagsrc, err.Error())
os.Exit(2)
}
return resp
}
func main() {
var fakeMap = map[string]string{
"site-web-100": "site-web-100",
"site-web-200": "site-web-200",
"site-web-300": "site-web-300",
"site-db-100": "site-db-100",
}
f, err := os.Create("/tmp/host.cfg")
if err != nil {
logr.LogLine(logr.Lerror, ltagsrc, err.Error())
}
defer f.Close()
hostGroups := make(map[string][]string)
//take a canned map of hosts, generate host and hostgroup files
for host := range fakeMap {
// for each hostey in the map
// write out a hostdef
// append hostname to a group list
f.WriteString(fmt.Sprintf(HostDef, host, host, host))
group := extractGroup(host)
fmt.Printf("%s\n", group)
hostGroups[group] = append(hostGroups[group], host)
}
// at the end, write out the group file using the group list
logr.LogLine(logr.Linfo, ltagsrc, fmt.Sprintf("%v", hostGroups))
f1, err := os.Create("/tmp/groups.cfg")
if err != nil {
logr.LogLine(logr.Lerror, ltagsrc, err.Error())
}
defer f1.Close()
// now print out the group file
for k := range hostGroups {
sHosts := strings.Join(hostGroups[k], ",")
logr.LogLine(logr.Linfo, ltagsrc, fmt.Sprintf("group: %s, hosts: %s\n", k, sHosts))
f1.WriteString(fmt.Sprintf(GroupDef, k, k, sHosts))
}
}
// DumpServices is a utility method that dumps all contents of etcd that match
// a specified base string
func DumpServices(kapi client.KeysAPI, baseStr string) {
//baseStr := "/site"
resp := ClientGet(kapi, baseStr)
// get the list of host type
for _, n := range resp.Node.Nodes {
resp1 := ClientGet(kapi, n.Key)
for _, n1 := range resp1.Node.Nodes {
logr.LogLine(logr.Linfo, ltagsrc, fmt.Sprintf("%s: %s", n1.Key, n1.Value))
}
}
}
func RestartNagios() {
// first confirm that we didn't blow the syntax
if _, err := execCmdOutput(nagiosCheckCmd, nagiosCheckArgs); err != nil {
logr.LogLine(logr.Lerror, ltagsrc, "check nagios config failed")
return
}
logr.LogLine(logr.Linfo, ltagsrc, "check nagios succeeded")
// then restart
_, err := execCmdOutput(nagiosPkillCmd, nagiosPkillArgs)
if err != nil {
logr.LogLine(logr.Lerror, ltagsrc, "pkill HUP nagios failed")
return
}
/*logr.LogLine(logr.Linfo, ltagsrc, fmt.Sprintf("got nagios pid: %s", pid))
useArgs := append(nagiosHUPArgs, pid)
if _, err := execCmdOutput(nagiosHUPCmd, useArgs); err != nil {
logr.LogLine(logr.Lerror, ltagsrc, "HUP nagios failed")
return
}*/
logr.LogLine(logr.Linfo, ltagsrc, "nagios restarted")
}
// dump hostmap out to a file
func writeHostMap(hostMap map[string]string) {
if host_list_file == "" {
return
}
f, err := os.Create(host_list_file)
if err != nil {
logr.LogLine(logr.Lerror, ltagsrc, err.Error())
}
defer f.Close()
for host := range hostMap {
f.WriteString(fmt.Sprintf("%s\n", host))
}
}
func GetEtcdKapi(serverList []string) (client.KeysAPI, error) {
cfg := client.Config{
Endpoints: serverList,
Transport: client.DefaultTransport,
// set timeout per request to fail fast when the target endpoint is unavailable
HeaderTimeoutPerRequest: time.Second,
}
c, err := client.New(cfg)
if err != nil {
logr.LogLine(logr.Lerror, ltagsrc, err.Error())
return nil, err
}
return client.NewKeysAPI(c), nil
}
// GenerateFiles takes the source host map and writes out a host and group nagios config file
// to the path passed to the function.
func GenerateFiles(hdMap map[string]string, hostPath string, groupPath string) {
f, err := os.Create(hostPath)
if err != nil {
logr.LogLine(logr.Lerror, ltagsrc, err.Error())
}
defer f.Close()
hostGroups := make(map[string][]string)
hostlist := make([]string, 0, len(hdMap))
for host := range hdMap {
hostlist = append(hostlist, host)
}
sort.Strings(hostlist)
for _, h := range hostlist {
// for each hostey in the map
// write out a hostdef
// append hostname to a group list
f.WriteString(fmt.Sprintf(HostDef, h, h, h))
group := extractGroup(h)
hostGroups[group] = append(hostGroups[group], h)
}
// at the end, write out the group file using the group list
f1, err := os.Create(groupPath)
if err != nil {
logr.LogLine(logr.Lerror, ltagsrc, err.Error())
}
defer f1.Close()
// now print out the group file
for k := range hostGroups {
sHosts := strings.Join(hostGroups[k], ",")
f1.WriteString(fmt.Sprintf(GroupDef, k, k, sHosts))
}
go RestartNagios()
}
func DeleteFromMap(k string) {
logr.LogLine(logr.Linfo, ltagsrc, fmt.Sprintf("deleting key: %s", k))
delete(hostMap, k)
}
func main() {
// handle command line args
var configFile string
flag.StringVar(&configFile, "cfg", "daemon.cfg", "full path to daemon config")
var logConfigPath string
flag.StringVar(&logConfigPath, "logcfg", "none", "full path to log config")
flag.Parse()
if logConfigPath != "none" {
logr.SetConfig(logConfigPath)
}
config, err := config.ParseConfig(configFile)
if err != nil {
log.Fatal("couldn't open config file %s", configFile, err)
}
nagios_host_file = config["nagios_host_file"]
nagios_group_file = config["nagios_groups_file"]
host_list_file = config["host_list_file"]
watch_root := config["etcd_watch_root_url"]
s := config["file_rewrite_interval"]
if s != "" {
i, err := strconv.Atoi(s)
if err != nil {
logr.LogLine(logr.Linfo, ltagsrc, fmt.Sprintf("invalid file rewrite val in config: ", err))
}
if i != 0 {
fileRewriteInterval = i
}
}
logr.LogLine(logr.Linfo, ltagsrc, fmt.Sprintf("file rewrite interval set to: %d", fileRewriteInterval))
// expect this to be csv or single entry
etcd_server_list := strings.Split(config["etcd_server_list"], ",")
kapi, err := GetEtcdKapi(etcd_server_list)
if err != nil {
// we die on the inital because it assumes a user is there watching
logr.LogLine(logr.Lfatal, ltagsrc, fmt.Sprintf("Error getting etcdKAPI", err.Error()))
os.Exit(2)
}
logr.LogLine(logr.Linfo, ltagsrc, "got client")
etcdWatcher.InitDataMap(kapi, watch_root)
logr.LogLine(logr.Linfo, ltagsrc, "Dumping map contents for verification")
etcdWatcher.DumpMap()
logr.LogLine(logr.Linfo, ltagsrc, "Generating initial config files")
regenHosts()
//
// spin up the web server
//
go webservice.StartWebService(config["web_listen_port"])
watcherOpts := client.WatcherOptions{AfterIndex: 0, Recursive: true}
w := kapi.Watcher(watch_root, &watcherOpts)
logr.LogLine(logr.Linfo, ltagsrc, "Waiting for an update...")
restartCount := 0
for {
r, err := w.Next(context.Background())
if err != nil {
logr.LogLine(logr.Lerror, ltagsrc, fmt.Sprintf("Error watching etcd", err.Error()))
// has etcd gone away?
restartCount++
switch {
case restartCount < 10:
logr.LogLine(logr.Lerror, ltagsrc, "Sleeping for 10 seconds then retrying")
time.Sleep(10 * time.Second)
case restartCount < 20:
time.Sleep(30 * time.Second)
logr.LogLine(logr.Lerror, ltagsrc, "Sleeping for 30 seconds then retrying.")
default:
time.Sleep(60 * time.Second * 5) // default sleep 5 minutes before retry
logr.LogLine(logr.Lerror, ltagsrc, "Sleeping for 5 minutes then retrying.")
}
kapi, err := GetEtcdKapi(etcd_server_list)
if err != nil {
logr.LogLine(logr.Lerror, ltagsrc, fmt.Sprintf("Error getting etcdKAPI", err.Error()))
}
w = kapi.Watcher(watch_root, &watcherOpts)
continue
}
// do something with it here
action := r.Action
k := r.Node.Key
v := r.Node.Value
switch action {
case "delete":
logr.LogLine(logr.Linfo, ltagsrc, fmt.Sprintf("delete of key: %s", k))
go removeHost(k)
case "set":
logr.LogLine(logr.Linfo, ltagsrc, fmt.Sprintf("update of key: %s, value: %s", k, v))
go updateHost(k, v)
}
}
}
func removeHost(k string) {
logr.LogLine(logr.Linfo, ltagsrc, fmt.Sprintf("removeHost in daemon.go -- k:%s", k))
regenHosts()
}
// DumpMap walks the host map and dumps out key-value pairs
func DumpMap() {
for k, v := range hostMap {
logr.LogLine(logr.Linfo, ltagsrc, fmt.Sprintf("%s: %+v", k, v))
}
}