func (m *EtcdManager) WatchNetworks(ctx context.Context, cursor interface{}) (NetworkWatchResult, error) { if cursor == nil { return m.networkWatchReset(ctx) } nextIndex, err := getNextIndex(cursor) if err != nil { return NetworkWatchResult{}, err } DoWatch: resp, err := m.registry.watch(ctx, "", nextIndex) switch { case err == nil: result, err, again := m.parseNetworkWatchResponse(resp) if again { nextIndex = resp.Node.ModifiedIndex goto DoWatch } return result, err case isIndexTooSmall(err): log.Warning("Watch of subnet leases failed because etcd index outside history window") return m.networkWatchReset(ctx) default: return NetworkWatchResult{}, err } }
func (m *EtcdManager) WatchLeases(ctx context.Context, network string, cursor interface{}) (WatchResult, error) { if cursor == nil { return m.watchReset(ctx, network) } nextIndex := uint64(0) if wc, ok := cursor.(watchCursor); ok { nextIndex = wc.index } else if s, ok := cursor.(string); ok { var err error nextIndex, err = strconv.ParseUint(s, 10, 64) if err != nil { return WatchResult{}, fmt.Errorf("failed to parse cursor: %v", err) } } else { return WatchResult{}, fmt.Errorf("internal error: watch cursor is of unknown type") } resp, err := m.registry.watchSubnets(ctx, network, nextIndex) switch { case err == nil: return parseSubnetWatchResponse(resp) case isIndexTooSmall(err): log.Warning("Watch of subnet leases failed because etcd index outside history window") return m.watchReset(ctx, network) default: return WatchResult{}, err } }
func (m *LocalManager) WatchNetworks(ctx context.Context, cursor interface{}) (NetworkWatchResult, error) { if cursor == nil { return m.networkWatchReset(ctx) } nextIndex, err := getNextIndex(cursor) if err != nil { return NetworkWatchResult{}, err } for { evt, index, err := m.registry.watchNetworks(ctx, nextIndex) switch { case err == nil: return NetworkWatchResult{ Events: []Event{evt}, Cursor: watchCursor{index}, }, nil case err == errTryAgain: nextIndex = index case isIndexTooSmall(err): log.Warning("Watch of networks failed because etcd index outside history window") return m.networkWatchReset(ctx) default: return NetworkWatchResult{}, err } } }
func (m *LocalManager) WatchLeases(ctx context.Context, network string, cursor interface{}) (LeaseWatchResult, error) { if cursor == nil { return m.leasesWatchReset(ctx, network) } nextIndex, err := getNextIndex(cursor) if err != nil { return LeaseWatchResult{}, err } evt, index, err := m.registry.watchSubnets(ctx, network, nextIndex) switch { case err == nil: return LeaseWatchResult{ Events: []Event{evt}, Cursor: watchCursor{index}, }, nil case isIndexTooSmall(err): log.Warning("Watch of subnet leases failed because etcd index outside history window") return m.leasesWatchReset(ctx, network) default: return LeaseWatchResult{}, err } }
func (m *Manager) Run(ctx context.Context) { wg := sync.WaitGroup{} if m.isMultiNetwork() { for { // Try adding initial networks result, err := m.sm.WatchNetworks(ctx, nil) if err == nil { for _, n := range result.Snapshot { if m.isNetAllowed(n) { m.networks[n] = NewNetwork(ctx, m.sm, m.bm, n, m.ipMasq) } } break } // Otherwise retry in a few seconds log.Warning("Failed to retrieve networks (will retry): %v", err) select { case <-ctx.Done(): return case <-time.After(time.Second): } } } else { m.networks[""] = NewNetwork(ctx, m.sm, m.bm, "", m.ipMasq) } // Run existing networks m.forEachNetwork(func(n *Network) { wg.Add(1) go func(n *Network) { m.runNetwork(n) wg.Done() }(n) }) if opts.watchNetworks { m.watchNetworks() } wg.Wait() m.bm.Wait() }
func (sm *SubnetManager) parseSubnetWatchError(err error) (batch *EventBatch, out error) { etcdErr, ok := err.(*etcd.EtcdError) if ok && etcdErr.ErrorCode == etcdEventIndexCleared { // etcd maintains a history window for events and it's possible to fall behind. // to recover, get the current state and then "diff" against our cache to generate // events for the caller log.Warning("Watch of subnet leases failed because etcd index outside history window") leases, err := sm.getLeases() if err == nil { lb := sm.applyLeases(leases) batch = &lb } else { out = fmt.Errorf("Failed to retrieve subnet leases: %v", err) } } else { out = fmt.Errorf("Watch of subnet leases failed: %v", err) } return }
func (m *EtcdManager) WatchLeases(ctx context.Context, network string, cursor interface{}) (LeaseWatchResult, error) { if cursor == nil { return m.leaseWatchReset(ctx, network) } nextIndex, err := getNextIndex(cursor) if err != nil { return LeaseWatchResult{}, err } resp, err := m.registry.watch(ctx, path.Join(network, "subnets"), nextIndex) switch { case err == nil: return parseSubnetWatchResponse(resp) case isIndexTooSmall(err): log.Warning("Watch of subnet leases failed because etcd index outside history window") return m.leaseWatchReset(ctx, network) default: return LeaseWatchResult{}, err } }
func (n *Network) runOnce(extIface *backend.ExternalInterface, inited func(bn backend.Network)) error { if err := n.retryInit(); err != nil { return errCanceled } inited(n.bn) ctx, interruptFunc := context.WithCancel(n.ctx) wg := sync.WaitGroup{} wg.Add(1) go func() { n.bn.Run(ctx) wg.Done() }() evts := make(chan subnet.Event) wg.Add(1) go func() { subnet.WatchLease(ctx, n.sm, n.Name, n.bn.Lease().Subnet, evts) wg.Done() }() defer func() { if n.ipMasq { if err := teardownIPMasq(n.Config.Network); err != nil { log.Errorf("Failed to tear down IP Masquerade for network %v: %v", n.Name, err) } } }() defer wg.Wait() dur := n.bn.Lease().Expiration.Sub(time.Now()) - renewMargin for { select { case <-time.After(dur): err := n.sm.RenewLease(n.ctx, n.Name, n.bn.Lease()) if err != nil { log.Error("Error renewing lease (trying again in 1 min): ", err) dur = time.Minute continue } log.Info("Lease renewed, new expiration: ", n.bn.Lease().Expiration) dur = n.bn.Lease().Expiration.Sub(time.Now()) - renewMargin case e := <-evts: switch e.Type { case subnet.EventAdded: n.bn.Lease().Expiration = e.Lease.Expiration dur = n.bn.Lease().Expiration.Sub(time.Now()) - renewMargin case subnet.EventRemoved: log.Warning("Lease has been revoked") interruptFunc() return errInterrupted } case <-n.ctx.Done(): return errCanceled } } }