func run(be backend.Backend, exit chan int) { var err error defer func() { if err == nil || err == task.ErrCanceled { exit <- 0 } else { log.Error(err) exit <- 1 } }() iface, ipaddr, err := lookupIface() if err != nil { return } if iface.MTU == 0 { err = fmt.Errorf("Failed to determine MTU for %s interface", ipaddr) return } log.Infof("Using %s as external interface", ipaddr) sn, err := be.Init(iface, ipaddr, opts.ipMasq) if err != nil { log.Error("Could not init %v backend: %v", be.Name(), err) return } writeSubnetFile(sn) daemon.SdNotify("READY=1") log.Infof("%s mode initialized", be.Name()) be.Run() }
func (n *Network) Init(ctx context.Context, iface *net.Interface, iaddr net.IP, eaddr net.IP) *backend.SubnetDef { var be backend.Backend var sn *backend.SubnetDef steps := []func() error{ func() (err error) { n.Config, err = n.sm.GetNetworkConfig(ctx, n.Name) if err != nil { log.Error("Failed to retrieve network config: ", err) } return }, func() (err error) { be, err = newBackend(n.sm, n.Name, n.Config) if err != nil { log.Error("Failed to create backend: ", err) } else { n.be = be } return }, func() (err error) { sn, err = be.Init(ctx, iface, iaddr, eaddr) if err != nil { log.Errorf("Failed to initialize network %v (type %v): %v", n.Name, n.Config.BackendType, err) } n.lease = sn.Lease return }, func() (err error) { if n.ipMasq { flannelNet := n.Config.Network if err = setupIPMasq(flannelNet); err != nil { log.Errorf("Failed to set up IP Masquerade for network %v: %v", n.Name, err) } } return }, } for _, s := range steps { for ; ; time.Sleep(time.Second) { select { case <-ctx.Done(): return nil default: } err := s() if err == nil { break } } } return sn }
func (vb *VXLANBackend) handleInitialSubnetEvents(batch []subnet.Event) error { log.Infof("Handling initial subnet events") fdbTable, err := vb.dev.GetL2List() if err != nil { return fmt.Errorf("Error fetching L2 table: %v", err) } for _, fdbEntry := range fdbTable { log.Infof("fdb already populated with: %s %s ", fdbEntry.IP, fdbEntry.HardwareAddr) } evtMarker := make([]bool, len(batch)) leaseAttrsList := make([]vxlanLeaseAttrs, len(batch)) fdbEntryMarker := make([]bool, len(fdbTable)) for i, evt := range batch { if evt.Lease.Attrs.BackendType != "vxlan" { log.Warningf("Ignoring non-vxlan subnet: type=%v", evt.Lease.Attrs.BackendType) evtMarker[i] = true continue } if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &leaseAttrsList[i]); err != nil { log.Error("Error decoding subnet lease JSON: ", err) evtMarker[i] = true continue } for j, fdbEntry := range fdbTable { if evt.Lease.Attrs.PublicIP.ToIP().Equal(fdbEntry.IP) && bytes.Equal([]byte(leaseAttrsList[i].VtepMAC), []byte(fdbEntry.HardwareAddr)) { evtMarker[i] = true fdbEntryMarker[j] = true break } } vb.rts.set(evt.Lease.Subnet, net.HardwareAddr(leaseAttrsList[i].VtepMAC)) } for j, marker := range fdbEntryMarker { if !marker && fdbTable[j].IP != nil { err := vb.dev.DelL2(neigh{IP: ip.FromIP(fdbTable[j].IP), MAC: fdbTable[j].HardwareAddr}) if err != nil { log.Error("Delete L2 failed: ", err) } } } for i, marker := range evtMarker { if !marker { err := vb.dev.AddL2(neigh{IP: batch[i].Lease.Attrs.PublicIP, MAC: net.HardwareAddr(leaseAttrsList[i].VtepMAC)}) if err != nil { log.Error("Add L2 failed: ", err) } } } return nil }
func jsonResponse(w http.ResponseWriter, code int, v interface{}) { w.Header().Set("Content-Type", "application/json; charset=utf-8") w.WriteHeader(code) if err := json.NewEncoder(w).Encode(v); err != nil { log.Error("Error JSON encoding response: %v", err) } }
func (sm *SubnetManager) AcquireLease(extIP ip.IP4, data interface{}, cancel chan bool) (ip.IP4Net, error) { dataBytes, err := json.Marshal(data) if err != nil { return ip.IP4Net{}, err } var sn ip.IP4Net for { sn, err = sm.acquireLeaseOnce(extIP, string(dataBytes), cancel) switch { case err == nil: log.Info("Subnet lease acquired: ", sn) return sn, nil case err == task.ErrCanceled: return ip.IP4Net{}, err default: log.Error("Failed to acquire subnet: ", err) } select { case <-time.After(time.Second): case <-cancel: return ip.IP4Net{}, task.ErrCanceled } } }
func (m *EtcdManager) AcquireLease(ctx context.Context, network string, attrs *LeaseAttrs) (*Lease, error) { config, err := m.GetNetworkConfig(ctx, network) if err != nil { return nil, err } for { l, err := m.acquireLeaseOnce(ctx, network, config, attrs) switch { case err == nil: log.Info("Subnet lease acquired: ", l.Subnet) return l, nil case err == context.Canceled, err == context.DeadlineExceeded: return nil, err default: log.Error("Failed to acquire subnet: ", err) } select { case <-time.After(time.Second): case <-ctx.Done(): return nil, ctx.Err() } } }
func initAndRun(ctx context.Context, sm subnet.Manager, netnames []string) { iface, iaddr, err := lookupIface() if err != nil { log.Error(err) return } if iface.MTU == 0 { log.Errorf("Failed to determine MTU for %s interface", iaddr) return } var eaddr net.IP if len(opts.publicIP) > 0 { eaddr = net.ParseIP(opts.publicIP) } if eaddr == nil { eaddr = iaddr } log.Infof("Using %s as external interface", iaddr) log.Infof("Using %s as external endpoint", eaddr) nets := []*network.Network{} for _, n := range netnames { nets = append(nets, network.New(sm, n, opts.ipMasq)) } wg := sync.WaitGroup{} for _, n := range nets { go func(n *network.Network) { wg.Add(1) defer wg.Done() sn := n.Init(ctx, iface, iaddr, eaddr) if sn != nil { if isMultiNetwork() { path := filepath.Join(opts.subnetDir, n.Name) + ".env" if err := writeSubnetFile(path, n.Config.Network, sn); err != nil { return } } else { if err := writeSubnetFile(opts.subnetFile, n.Config.Network, sn); err != nil { return } daemon.SdNotify("READY=1") } n.Run(ctx) log.Infof("%v exited", n.Name) } }(n) } wg.Wait() }
func (sm *SubnetManager) acquireLeaseOnce(extIP ip.IP4, data string, cancel chan bool) (ip.IP4Net, error) { for i := 0; i < registerRetries; i++ { var err error sm.leases, err = sm.getLeases() if err != nil { return ip.IP4Net{}, err } // try to reuse a subnet if there's one that matches our IP for _, l := range sm.leases { var ba BaseAttrs err = json.Unmarshal([]byte(l.Data), &ba) if err != nil { log.Error("Error parsing subnet lease JSON: ", err) } else { if extIP == ba.PublicIP { resp, err := sm.registry.updateSubnet(l.Network.StringSep(".", "-"), data, subnetTTL) if err != nil { return ip.IP4Net{}, err } sm.myLease.Network = l.Network sm.leaseExp = *resp.Node.Expiration return l.Network, nil } } } // no existing match, grab a new one sn, err := sm.allocateSubnet() if err != nil { return ip.IP4Net{}, err } resp, err := sm.registry.createSubnet(sn.StringSep(".", "-"), data, subnetTTL) switch { case err == nil: sm.myLease.Network = sn sm.leaseExp = *resp.Node.Expiration return sn, nil // if etcd returned Key Already Exists, try again. case err.(*etcd.EtcdError).ErrorCode == etcdKeyAlreadyExists: break default: return ip.IP4Net{}, err } // before moving on, check for cancel if interrupted(cancel) { return ip.IP4Net{}, task.ErrCanceled } } return ip.IP4Net{}, errors.New("Max retries reached trying to acquire a subnet") }
func (vb *VXLANBackend) handleSubnetEvents(batch []subnet.Event) { for _, evt := range batch { switch evt.Type { case subnet.EventAdded: log.Info("Subnet added: ", evt.Lease.Subnet) if evt.Lease.Attrs.BackendType != "vxlan" { log.Warningf("Ignoring non-vxlan subnet: type=%v", evt.Lease.Attrs.BackendType) continue } var attrs vxlanLeaseAttrs if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &attrs); err != nil { log.Error("Error decoding subnet lease JSON: ", err) continue } vb.rts.set(evt.Lease.Subnet, net.HardwareAddr(attrs.VtepMAC)) vb.dev.AddL2(neigh{IP: evt.Lease.Attrs.PublicIP, MAC: net.HardwareAddr(attrs.VtepMAC)}) case subnet.EventRemoved: log.Info("Subnet removed: ", evt.Lease.Subnet) if evt.Lease.Attrs.BackendType != "vxlan" { log.Warningf("Ignoring non-vxlan subnet: type=%v", evt.Lease.Attrs.BackendType) continue } var attrs vxlanLeaseAttrs if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &attrs); err != nil { log.Error("Error decoding subnet lease JSON: ", err) continue } if len(attrs.VtepMAC) > 0 { vb.dev.DelL2(neigh{IP: evt.Lease.Attrs.PublicIP, MAC: net.HardwareAddr(attrs.VtepMAC)}) } vb.rts.remove(evt.Lease.Subnet) default: log.Error("Internal error: unknown event type: ", int(evt.Type)) } } }
func makeSubnetManager() *subnet.SubnetManager { for { sm, err := subnet.NewSubnetManager(opts.etcdEndpoint, opts.etcdPrefix) if err == nil { return sm } log.Error("Failed to create SubnetManager: ", err) time.Sleep(time.Second) } }
func (rb *HostgwBackend) handleSubnetEvents(batch []subnet.Event) { for _, evt := range batch { switch evt.Type { case subnet.SubnetAdded: log.Infof("Subnet added: %v via %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP) if evt.Lease.Attrs.BackendType != "host-gw" { log.Warningf("Ignoring non-host-gw subnet: type=%v", evt.Lease.Attrs.BackendType) continue } route := netlink.Route{ Dst: evt.Lease.Subnet.ToIPNet(), Gw: evt.Lease.Attrs.PublicIP.ToIP(), LinkIndex: rb.extIface.Index, } if rb.extIaddr.Equal(route.Gw) { continue } if err := netlink.RouteAdd(&route); err != nil { log.Errorf("Error adding route to %v via %v: %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, err) continue } rb.addToRouteList(route) case subnet.SubnetRemoved: log.Info("Subnet removed: ", evt.Lease.Subnet) if evt.Lease.Attrs.BackendType != "host-gw" { log.Warningf("Ignoring non-host-gw subnet: type=%v", evt.Lease.Attrs.BackendType) continue } route := netlink.Route{ Dst: evt.Lease.Subnet.ToIPNet(), Gw: evt.Lease.Attrs.PublicIP.ToIP(), LinkIndex: rb.extIface.Index, } if err := netlink.RouteDel(&route); err != nil { log.Errorf("Error deleting route to %v: %v", evt.Lease.Subnet, err) continue } rb.removeFromRouteList(route) default: log.Error("Internal error: unknown event type: ", int(evt.Type)) } } }
func (m *UdpBackend) processSubnetEvents(batch subnet.EventBatch) { for _, evt := range batch { switch evt.Type { case subnet.SubnetAdded: log.Info("Subnet added: ", evt.Lease.Network) var attrs subnet.BaseAttrs if err := json.Unmarshal([]byte(evt.Lease.Data), &attrs); err != nil { log.Error("Error decoding subnet lease JSON: ", err) continue } setRoute(m.ctl, evt.Lease.Network, attrs.PublicIP, m.cfg.Port) case subnet.SubnetRemoved: log.Info("Subnet removed: ", evt.Lease.Network) removeRoute(m.ctl, evt.Lease.Network) default: log.Error("Internal error: unknown event type: ", int(evt.Type)) } } }
func (n *Network) retryInit() error { for { err := n.init() if err == nil || err == context.Canceled { return err } log.Error(err) select { case <-n.ctx.Done(): return n.ctx.Err() case <-time.After(time.Second): } } }
func (vb *VXLANBackend) Run() { vb.wg.Add(1) go func() { subnet.LeaseRenewer(vb.ctx, vb.sm, vb.network, vb.lease) log.Info("LeaseRenewer exited") vb.wg.Done() }() log.Info("Watching for L3 misses") misses := make(chan *netlink.Neigh, 100) // Unfrtunately MonitorMisses does not take a cancel channel // as there's no wait to interrupt netlink socket recv go vb.dev.MonitorMisses(misses) log.Info("Watching for new subnet leases") evts := make(chan []subnet.Event) vb.wg.Add(1) go func() { subnet.WatchLeases(vb.ctx, vb.sm, vb.network, vb.lease, evts) log.Info("WatchLeases exited") vb.wg.Done() }() defer vb.wg.Wait() initialEvtsBatch := <-evts for { err := vb.handleInitialSubnetEvents(initialEvtsBatch) if err == nil { break } log.Error(err, " About to retry") time.Sleep(time.Second) } for { select { case miss := <-misses: vb.handleMiss(miss) case evtBatch := <-evts: vb.handleSubnetEvents(evtBatch) case <-vb.ctx.Done(): return } } }
func (n *Network) Run(extIface *backend.ExternalInterface, inited func(bn backend.Network)) { wg := sync.WaitGroup{} For: for { err := n.init() switch err { case nil: break For case context.Canceled: return default: log.Error(err) select { case <-n.ctx.Done(): return case <-time.After(time.Second): } } } inited(n.bn) wg.Add(1) go func() { n.bn.Run(n.ctx) wg.Done() }() wg.Add(1) go func() { subnet.LeaseRenewer(n.ctx, n.sm, n.Name, n.bn.Lease()) wg.Done() }() defer func() { if n.ipMasq { if err := teardownIPMasq(n.Config.Network); err != nil { log.Errorf("Failed to tear down IP Masquerade for network %v: %v", n.Name, err) } } }() wg.Wait() }
func (m *UdpBackend) processSubnetEvents(batch []subnet.Event) { for _, evt := range batch { switch evt.Type { case subnet.EventAdded: log.Info("Subnet added: ", evt.Lease.Subnet) setRoute(m.ctl, evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, m.cfg.Port) case subnet.EventRemoved: log.Info("Subnet removed: ", evt.Lease.Subnet) removeRoute(m.ctl, evt.Lease.Subnet) default: log.Error("Internal error: unknown event type: ", int(evt.Type)) } } }
func (n *network) Run(ctx context.Context) { log.Info("Watching for L3 misses") misses := make(chan *netlink.Neigh, 100) // Unfrtunately MonitorMisses does not take a cancel channel // as there's no wait to interrupt netlink socket recv go n.dev.MonitorMisses(misses) wg := sync.WaitGroup{} log.Info("Watching for new subnet leases") evts := make(chan []subnet.Event) wg.Add(1) go func() { subnet.WatchLeases(ctx, n.sm, n.name, n.SubnetLease, evts) log.Info("WatchLeases exited") wg.Done() }() defer wg.Wait() initialEvtsBatch := <-evts for { err := n.handleInitialSubnetEvents(initialEvtsBatch) if err == nil { break } log.Error(err, " About to retry") time.Sleep(time.Second) } for { select { case miss := <-misses: n.handleMiss(miss) case evtBatch := <-evts: n.handleSubnetEvents(evtBatch) case <-ctx.Done(): return } } }
func (dev *vxlanDevice) MonitorMisses(misses chan *netlink.Neigh) { nlsock, err := nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_NEIGH) if err != nil { log.Error("Failed to subscribe to netlink RTNLGRP_NEIGH messages") return } for { msgs, err := nlsock.Receive() if err != nil { log.Errorf("Failed to receive from netlink: %v ", err) time.Sleep(1 * time.Second) continue } for _, msg := range msgs { dev.processNeighMsg(msg, misses) } } }
func LeaseRenewer(ctx context.Context, m Manager, network string, lease *Lease) { dur := lease.Expiration.Sub(clock.Now()) - renewMargin for { select { case <-time.After(dur): err := m.RenewLease(ctx, network, lease) if err != nil { log.Error("Error renewing lease (trying again in 1 min): ", err) dur = time.Minute continue } log.Info("Lease renewed, new expiration: ", lease.Expiration) dur = lease.Expiration.Sub(clock.Now()) - renewMargin case <-ctx.Done(): return } } }
func runCProxy(tun *os.File, conn *net.UDPConn, ctl *os.File, tunIP ip.IP4, tunMTU int) { var log_errors int if log.V(1) { log_errors = 1 } c, err := conn.File() if err != nil { log.Error("Converting UDPConn to File failed: ", err) return } defer c.Close() C.run_proxy( C.int(tun.Fd()), C.int(c.Fd()), C.int(ctl.Fd()), C.in_addr_t(tunIP.NetworkOrder()), C.size_t(tunMTU), C.int(log_errors), ) }
func (dev *vxlanDevice) processNeighMsg(msg syscall.NetlinkMessage, misses chan *netlink.Neigh) { neigh, err := netlink.NeighDeserialize(msg.Data) if err != nil { log.Error("Failed to deserialize netlink ndmsg: %v", err) return } if int(neigh.LinkIndex) != dev.link.Index { return } if msg.Header.Type != syscall.RTM_GETNEIGH && msg.Header.Type != syscall.RTM_NEWNEIGH { return } if !isNeighResolving(neigh.State) { // misses come with NUD_STALE bit set return } misses <- neigh }
func (sm *SubnetManager) LeaseRenewer(cancel chan bool) { dur := sm.leaseExp.Sub(time.Now()) - renewMargin for { select { case <-time.After(dur): resp, err := sm.registry.updateSubnet(sm.myLease.Network.StringSep(".", "-"), sm.myLease.Data, subnetTTL) if err != nil { log.Error("Error renewing lease (trying again in 1 min): ", err) dur = time.Minute continue } sm.leaseExp = *resp.Node.Expiration log.Info("Lease renewed, new expiration: ", sm.leaseExp) dur = sm.leaseExp.Sub(time.Now()) - renewMargin case <-cancel: return } } }
func (n *Network) runOnce(extIface *backend.ExternalInterface, inited func(bn backend.Network)) error { if err := n.retryInit(); err != nil { return errCanceled } inited(n.bn) ctx, interruptFunc := context.WithCancel(n.ctx) wg := sync.WaitGroup{} wg.Add(1) go func() { n.bn.Run(ctx) wg.Done() }() evts := make(chan subnet.Event) wg.Add(1) go func() { subnet.WatchLease(ctx, n.sm, n.Name, n.bn.Lease().Subnet, evts) wg.Done() }() defer func() { if n.ipMasq { if err := teardownIPMasq(n.Config.Network); err != nil { log.Errorf("Failed to tear down IP Masquerade for network %v: %v", n.Name, err) } } }() defer wg.Wait() dur := n.bn.Lease().Expiration.Sub(time.Now()) - renewMargin for { select { case <-time.After(dur): err := n.sm.RenewLease(n.ctx, n.Name, n.bn.Lease()) if err != nil { log.Error("Error renewing lease (trying again in 1 min): ", err) dur = time.Minute continue } log.Info("Lease renewed, new expiration: ", n.bn.Lease().Expiration) dur = n.bn.Lease().Expiration.Sub(time.Now()) - renewMargin case e := <-evts: switch e.Type { case subnet.EventAdded: n.bn.Lease().Expiration = e.Lease.Expiration dur = n.bn.Lease().Expiration.Sub(time.Now()) - renewMargin case subnet.EventRemoved: log.Warning("Lease has been revoked") interruptFunc() return errInterrupted } case <-n.ctx.Done(): return errCanceled } } }
func (vb *VXLANBackend) Init(extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (*backend.SubnetDef, error) { // Parse our configuration if len(vb.config.Backend) > 0 { if err := json.Unmarshal(vb.config.Backend, &vb.cfg); err != nil { return nil, fmt.Errorf("error decoding VXLAN backend config: %v", err) } } devAttrs := vxlanDeviceAttrs{ vni: uint32(vb.cfg.VNI), name: fmt.Sprintf("flannel.%v", vb.cfg.VNI), vtepIndex: extIface.Index, vtepAddr: extIaddr, vtepPort: vb.cfg.Port, } var err error for { vb.dev, err = newVXLANDevice(&devAttrs) if err == nil { break } else { log.Error("VXLAN init: ", err) log.Info("Retrying in 1 second...") // wait 1 sec before retrying time.Sleep(1 * time.Second) } } sa, err := newSubnetAttrs(extEaddr, vb.dev.MACAddr()) if err != nil { return nil, err } l, err := vb.sm.AcquireLease(vb.ctx, vb.network, sa) switch err { case nil: vb.lease = l case context.Canceled, context.DeadlineExceeded: return nil, err default: return nil, fmt.Errorf("failed to acquire lease: %v", err) } // vxlan's subnet is that of the whole overlay network (e.g. /16) // and not that of the individual host (e.g. /24) vxlanNet := ip.IP4Net{ IP: l.Subnet.IP, PrefixLen: vb.config.Network.PrefixLen, } if err = vb.dev.Configure(vxlanNet); err != nil { return nil, err } return &backend.SubnetDef{ Net: l.Subnet, MTU: vb.dev.MTU(), }, nil }
func main() { // glog will log to tmp files by default. override so all entries // can flow into journald (if running under systemd) flag.Set("logtostderr", "true") // now parse command line args flag.Parse() if flag.NArg() > 0 || opts.help { fmt.Fprintf(os.Stderr, "Usage: %s [OPTION]...\n", os.Args[0]) flag.PrintDefaults() os.Exit(0) } if opts.version { fmt.Fprintln(os.Stderr, Version) os.Exit(0) } flagsFromEnv("FLANNELD", flag.CommandLine) sm, err := newSubnetManager() if err != nil { log.Error("Failed to create SubnetManager: ", err) os.Exit(1) } var runFunc func(ctx context.Context) if opts.listen != "" { if opts.remote != "" { log.Error("--listen and --remote are mutually exclusive") os.Exit(1) } log.Info("running as server") runFunc = func(ctx context.Context) { remote.RunServer(ctx, sm, opts.listen, opts.remoteCAFile, opts.remoteCertfile, opts.remoteKeyfile) } } else { networks := strings.Split(opts.networks, ",") if len(networks) == 0 { networks = append(networks, "") } runFunc = func(ctx context.Context) { initAndRun(ctx, sm, networks) } } // Register for SIGINT and SIGTERM log.Info("Installing signal handlers") sigs := make(chan os.Signal, 1) signal.Notify(sigs, os.Interrupt, syscall.SIGTERM) ctx, cancel := context.WithCancel(context.Background()) wg := sync.WaitGroup{} wg.Add(1) go func() { runFunc(ctx) wg.Done() }() <-sigs // unregister to get default OS nuke behaviour in case we don't exit cleanly signal.Stop(sigs) log.Info("Exiting...") cancel() wg.Wait() }