// note: logging should be delegated somehow
func (r Replicator) updateOne(node string, done chan<- string, errCh chan<- error, quitCh <-chan struct{}) {
targetSHA, _ := r.Manifest.SHA()
nodeLogger := r.Logger.SubLogger(logrus.Fields{"node": node})
nodeLogger.WithField("sha", targetSHA).Infoln("Updating node")
_, err := r.Store.SetPod(kp.IntentPath(node, r.Manifest.ID()), r.Manifest)
for err != nil {
nodeLogger.WithField("err", err).Errorln("Could not write intent store")
errCh <- err
time.Sleep(1 * time.Second)
_, err = r.Store.SetPod(kp.IntentPath(node, r.Manifest.ID()), r.Manifest)
}
realityResults := make(chan kp.ManifestResult)
realityErr := make(chan error)
realityQuit := make(chan struct{})
defer close(realityQuit)
go r.Store.WatchPods(kp.RealityPath(node, r.Manifest.ID()), realityQuit, realityErr, realityResults)
REALITY_LOOP:
for {
select {
case <-quitCh:
return
case err := <-realityErr:
nodeLogger.WithField("err", err).Errorln("Could not read reality store")
errCh <- err
case mResult := <-realityResults:
receivedSHA, _ := mResult.Manifest.SHA()
if receivedSHA == targetSHA {
break REALITY_LOOP
} else {
nodeLogger.WithFields(logrus.Fields{"current": receivedSHA, "target": targetSHA}).Infoln("Waiting for current")
}
}
}
nodeLogger.NoFields().Infoln("Node is current")
healthResults := make(chan []health.Result)
healthErr := make(chan error)
healthQuit := make(chan struct{})
defer close(healthQuit)
go r.Health.WatchNodeService(node, r.Manifest.ID(), healthResults, healthErr, healthQuit)
HEALTH_LOOP:
for {
select {
case <-quitCh:
return
case err := <-healthErr:
nodeLogger.WithField("err", err).Errorln("Could not read health check")
errCh <- err
case res := <-healthResults:
id, status := health.FindWorst(res)
// treat an empty threshold as "passing"
threshold := health.Passing
if r.Threshold != "" {
threshold = r.Threshold
}
// is this status less than the threshold?
if health.Compare(status, threshold) < 0 {
nodeLogger.WithFields(logrus.Fields{"check": id, "health": status}).Infoln("Node is not healthy")
} else {
break HEALTH_LOOP
}
}
}
r.Logger.WithField("node", node).Infoln("Node is current and healthy")
select {
case done <- node:
case <-quitCh:
}
}
func main() {
kingpin.Version(version.VERSION)
kingpin.Parse()
opts := kp.Options{
Address: *consulUrl,
Token: *consulToken,
Client: net.NewHeaderClient(*headers, http.DefaultTransport),
HTTPS: *https,
}
store := kp.NewConsulStore(opts)
intents, _, err := store.ListPods(kp.INTENT_TREE)
if err != nil {
message := "Could not list intent kvpairs: %s"
if kvErr, ok := err.(kp.KVError); ok {
log.Fatalf(message, kvErr.UnsafeError)
} else {
log.Fatalf(message, err)
}
}
realities, _, err := store.ListPods(kp.REALITY_TREE)
if err != nil {
message := "Could not list reality kvpairs: %s"
if kvErr, ok := err.(kp.KVError); ok {
log.Fatalf(message, kvErr.UnsafeError)
} else {
log.Fatalf(message, err)
}
}
statusMap := make(map[string]map[string]inspect.NodePodStatus)
for _, kvp := range intents {
if inspect.AddKVPToMap(kvp, inspect.INTENT_SOURCE, *filterNodeName, *filterPodId, statusMap) != nil {
log.Fatal(err)
}
}
for _, kvp := range realities {
if inspect.AddKVPToMap(kvp, inspect.REALITY_SOURCE, *filterNodeName, *filterPodId, statusMap) != nil {
log.Fatal(err)
}
}
hchecker := health.NewConsulHealthChecker(opts)
for podId := range statusMap {
resultMap, err := hchecker.Service(podId)
if err != nil {
log.Fatalf("Could not retrieve health checks for pod %s: %s", podId, err)
}
for node, results := range resultMap {
if *filterNodeName != "" && node != *filterNodeName {
continue
}
old := statusMap[podId][node]
_, old.Health = health.FindWorst(results)
statusMap[podId][node] = old
}
}
enc := json.NewEncoder(os.Stdout)
enc.Encode(statusMap)
}