func (m *EtcdManager) WatchNetworks(ctx context.Context, cursor interface{}) (NetworkWatchResult, error) {
if cursor == nil {
return m.networkWatchReset(ctx)
}
nextIndex, err := getNextIndex(cursor)
if err != nil {
return NetworkWatchResult{}, err
}
DoWatch:
resp, err := m.registry.watch(ctx, "", nextIndex)
switch {
case err == nil:
result, err, again := m.parseNetworkWatchResponse(resp)
if again {
nextIndex = resp.Node.ModifiedIndex
goto DoWatch
}
return result, err
case isIndexTooSmall(err):
log.Warning("Watch of subnet leases failed because etcd index outside history window")
return m.networkWatchReset(ctx)
default:
return NetworkWatchResult{}, err
}
}
func (m *EtcdManager) WatchLeases(ctx context.Context, network string, cursor interface{}) (WatchResult, error) {
if cursor == nil {
return m.watchReset(ctx, network)
}
nextIndex := uint64(0)
if wc, ok := cursor.(watchCursor); ok {
nextIndex = wc.index
} else if s, ok := cursor.(string); ok {
var err error
nextIndex, err = strconv.ParseUint(s, 10, 64)
if err != nil {
return WatchResult{}, fmt.Errorf("failed to parse cursor: %v", err)
}
} else {
return WatchResult{}, fmt.Errorf("internal error: watch cursor is of unknown type")
}
resp, err := m.registry.watchSubnets(ctx, network, nextIndex)
switch {
case err == nil:
return parseSubnetWatchResponse(resp)
case isIndexTooSmall(err):
log.Warning("Watch of subnet leases failed because etcd index outside history window")
return m.watchReset(ctx, network)
default:
return WatchResult{}, err
}
}
func (m *LocalManager) WatchNetworks(ctx context.Context, cursor interface{}) (NetworkWatchResult, error) {
if cursor == nil {
return m.networkWatchReset(ctx)
}
nextIndex, err := getNextIndex(cursor)
if err != nil {
return NetworkWatchResult{}, err
}
for {
evt, index, err := m.registry.watchNetworks(ctx, nextIndex)
switch {
case err == nil:
return NetworkWatchResult{
Events: []Event{evt},
Cursor: watchCursor{index},
}, nil
case err == errTryAgain:
nextIndex = index
case isIndexTooSmall(err):
log.Warning("Watch of networks failed because etcd index outside history window")
return m.networkWatchReset(ctx)
default:
return NetworkWatchResult{}, err
}
}
}
func (m *LocalManager) WatchLeases(ctx context.Context, network string, cursor interface{}) (LeaseWatchResult, error) {
if cursor == nil {
return m.leasesWatchReset(ctx, network)
}
nextIndex, err := getNextIndex(cursor)
if err != nil {
return LeaseWatchResult{}, err
}
evt, index, err := m.registry.watchSubnets(ctx, network, nextIndex)
switch {
case err == nil:
return LeaseWatchResult{
Events: []Event{evt},
Cursor: watchCursor{index},
}, nil
case isIndexTooSmall(err):
log.Warning("Watch of subnet leases failed because etcd index outside history window")
return m.leasesWatchReset(ctx, network)
default:
return LeaseWatchResult{}, err
}
}
func (m *Manager) Run(ctx context.Context) {
wg := sync.WaitGroup{}
if m.isMultiNetwork() {
for {
// Try adding initial networks
result, err := m.sm.WatchNetworks(ctx, nil)
if err == nil {
for _, n := range result.Snapshot {
if m.isNetAllowed(n) {
m.networks[n] = NewNetwork(ctx, m.sm, m.bm, n, m.ipMasq)
}
}
break
}
// Otherwise retry in a few seconds
log.Warning("Failed to retrieve networks (will retry): %v", err)
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
}
} else {
m.networks[""] = NewNetwork(ctx, m.sm, m.bm, "", m.ipMasq)
}
// Run existing networks
m.forEachNetwork(func(n *Network) {
wg.Add(1)
go func(n *Network) {
m.runNetwork(n)
wg.Done()
}(n)
})
if opts.watchNetworks {
m.watchNetworks()
}
wg.Wait()
m.bm.Wait()
}
func (sm *SubnetManager) parseSubnetWatchError(err error) (batch *EventBatch, out error) {
etcdErr, ok := err.(*etcd.EtcdError)
if ok && etcdErr.ErrorCode == etcdEventIndexCleared {
// etcd maintains a history window for events and it's possible to fall behind.
// to recover, get the current state and then "diff" against our cache to generate
// events for the caller
log.Warning("Watch of subnet leases failed because etcd index outside history window")
leases, err := sm.getLeases()
if err == nil {
lb := sm.applyLeases(leases)
batch = &lb
} else {
out = fmt.Errorf("Failed to retrieve subnet leases: %v", err)
}
} else {
out = fmt.Errorf("Watch of subnet leases failed: %v", err)
}
return
}
func (m *EtcdManager) WatchLeases(ctx context.Context, network string, cursor interface{}) (LeaseWatchResult, error) {
if cursor == nil {
return m.leaseWatchReset(ctx, network)
}
nextIndex, err := getNextIndex(cursor)
if err != nil {
return LeaseWatchResult{}, err
}
resp, err := m.registry.watch(ctx, path.Join(network, "subnets"), nextIndex)
switch {
case err == nil:
return parseSubnetWatchResponse(resp)
case isIndexTooSmall(err):
log.Warning("Watch of subnet leases failed because etcd index outside history window")
return m.leaseWatchReset(ctx, network)
default:
return LeaseWatchResult{}, err
}
}
func (n *Network) runOnce(extIface *backend.ExternalInterface, inited func(bn backend.Network)) error {
if err := n.retryInit(); err != nil {
return errCanceled
}
inited(n.bn)
ctx, interruptFunc := context.WithCancel(n.ctx)
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
n.bn.Run(ctx)
wg.Done()
}()
evts := make(chan subnet.Event)
wg.Add(1)
go func() {
subnet.WatchLease(ctx, n.sm, n.Name, n.bn.Lease().Subnet, evts)
wg.Done()
}()
defer func() {
if n.ipMasq {
if err := teardownIPMasq(n.Config.Network); err != nil {
log.Errorf("Failed to tear down IP Masquerade for network %v: %v", n.Name, err)
}
}
}()
defer wg.Wait()
dur := n.bn.Lease().Expiration.Sub(time.Now()) - renewMargin
for {
select {
case <-time.After(dur):
err := n.sm.RenewLease(n.ctx, n.Name, n.bn.Lease())
if err != nil {
log.Error("Error renewing lease (trying again in 1 min): ", err)
dur = time.Minute
continue
}
log.Info("Lease renewed, new expiration: ", n.bn.Lease().Expiration)
dur = n.bn.Lease().Expiration.Sub(time.Now()) - renewMargin
case e := <-evts:
switch e.Type {
case subnet.EventAdded:
n.bn.Lease().Expiration = e.Lease.Expiration
dur = n.bn.Lease().Expiration.Sub(time.Now()) - renewMargin
case subnet.EventRemoved:
log.Warning("Lease has been revoked")
interruptFunc()
return errInterrupted
}
case <-n.ctx.Done():
return errCanceled
}
}
}