func (s consulStore) Delete(id rcf.ID) error {
key := kp.RollPath(id.String())
_, err := s.kv.Delete(key, nil)
if err != nil {
return consulutil.NewKVError("delete", key, err)
}
return nil
}
func (s consulStore) Lock(id rcf.ID, session string) (bool, error) {
key := kp.LockPath(kp.RollPath(id.String()))
success, _, err := s.kv.Acquire(&api.KVPair{
Key: key,
Value: []byte(session),
Session: session,
}, nil)
if err != nil {
return false, consulutil.NewKVError("acquire", key, err)
}
return success, nil
}
func (s consulStore) Put(u rollf.Update) error {
b, err := json.Marshal(u)
if err != nil {
return err
}
key := kp.RollPath(u.NewRC.String())
success, _, err := s.kv.CAS(&api.KVPair{
Key: kp.RollPath(u.NewRC.String()),
Value: b,
// it must not already exist
ModifyIndex: 0,
}, nil)
if err != nil {
return consulutil.NewKVError("cas", key, err)
}
if !success {
return fmt.Errorf("update with new RC ID %s already exists", u.NewRC)
}
return nil
}
// close one child
func (rlf *Farm) releaseChild(id fields.ID) {
rlf.logger.WithField("ru", id).Infoln("Releasing update")
close(rlf.children[id].quit)
delete(rlf.children, id)
// if our lock is active, attempt to gracefully release it
if rlf.lock != nil {
err := rlf.lock.Unlock(kp.LockPath(kp.RollPath(id.String())))
if err != nil {
rlf.logger.WithField("ru", id).Warnln("Could not release update lock")
}
}
}
func (s consulStore) Get(id rcf.ID) (rollf.Update, error) {
key := kp.RollPath(id.String())
kvp, _, err := s.kv.Get(key, nil)
if err != nil {
return rollf.Update{}, consulutil.NewKVError("get", key, err)
}
var ret rollf.Update
err = json.Unmarshal(kvp.Value, &ret)
if err != nil {
return rollf.Update{}, err
}
return ret, nil
}
// Start is a blocking function that monitors Consul for updates. The Farm will
// attempt to claim updates as they appear and, if successful, will start
// goroutines for those updatesto do their job. Closing the quit channel will
// cause this function to return, releasing all locks it holds.
//
// Start is not safe for concurrent execution. Do not execute multiple
// concurrent instances of Start.
func (rlf *Farm) Start(quit <-chan struct{}) {
subQuit := make(chan struct{})
defer close(subQuit)
rlWatch, rlErr := rlf.rls.Watch(subQuit)
START_LOOP:
for {
select {
case <-quit:
rlf.logger.NoFields().Infoln("Halt requested, releasing updates")
rlf.releaseChildren()
return
case session := <-rlf.sessions:
if session == "" {
// our session has expired, we must assume our locked children
// have all been released and that someone else may have
// claimed them by now
rlf.logger.NoFields().Errorln("Session expired, releasing updates")
rlf.lock = nil
rlf.releaseChildren()
} else {
// a new session has been acquired - only happens after an
// expiration message, so len(children)==0
rlf.logger.WithField("session", session).Infoln("Acquired new session")
lock := rlf.kps.NewUnmanagedLock(session, "")
rlf.lock = &lock
// TODO: restart the watch so that you get updates right away?
}
case err := <-rlErr:
rlf.logger.WithError(err).Errorln("Could not read consul updates")
case rlFields := <-rlWatch:
rlf.logger.WithField("n", len(rlFields)).Debugln("Received update update")
if rlf.lock == nil {
// we can't claim new nodes because our session is invalidated.
// raise an error and ignore this update
rlf.logger.NoFields().Warnln("Received update update, but do not have session to acquire locks")
continue
}
// track which children were found in the returned set
foundChildren := make(map[fields.ID]struct{})
for _, rlField := range rlFields {
rlLogger := rlf.logger.SubLogger(logrus.Fields{
"ru": rlField.NewRC,
})
rcField, err := rlf.rcs.Get(rlField.NewRC)
if err != nil {
rlLogger.WithError(err).Errorln("Could not read new RC")
continue
}
rlLogger = rlLogger.SubLogger(logrus.Fields{
"pod": rcField.Manifest.ID(),
})
if _, ok := rlf.children[rlField.NewRC]; ok {
// this one is already ours, skip
rlLogger.NoFields().Debugln("Got update already owned by self")
foundChildren[rlField.NewRC] = struct{}{}
continue
}
err = rlf.lock.Lock(kp.LockPath(kp.RollPath(rlField.NewRC.String())))
if _, ok := err.(kp.AlreadyLockedError); ok {
// someone else must have gotten it first - log and move to
// the next one
rlLogger.NoFields().Debugln("Lock on update was denied")
continue
} else if err != nil {
rlLogger.NoFields().Errorln("Got error while locking update - session may be expired")
// stop processing this update and go back to the select
// chances are this error is a network problem or session
// expiry, and all the others in this update would also fail
continue START_LOOP
}
// at this point the ru is ours, time to spin it up
rlLogger.NoFields().Infoln("Acquired lock on new update, spawning")
newChild := rlf.factory.New(rlField, rlLogger, *rlf.lock)
childQuit := make(chan struct{})
rlf.children[rlField.NewRC] = childRU{ru: newChild, quit: childQuit}
foundChildren[rlField.NewRC] = struct{}{}
go func(id fields.ID) {
if !newChild.Run(childQuit) {
// returned false, farm must have asked us to quit
return
}
// our lock on this RU won't be released until it's deleted,
// so if we fail to delete it, we have to retry
for err := rlf.rls.Delete(id); err != nil; err = rlf.rls.Delete(id) {
rlLogger.WithError(err).Errorln("Could not delete update")
time.Sleep(1 * time.Second)
}
}(rlField.NewRC) // do not close over rlField, it's a loop variable
}
// now remove any children that were not found in the result set
rlf.logger.NoFields().Debugln("Pruning updates that have disappeared")
for id := range rlf.children {
if _, ok := foundChildren[id]; !ok {
rlf.releaseChild(id)
}
}
}
}
}
