func RunServer(ctx context.Context, sm subnet.Manager, listenAddr, cafile, certfile, keyfile string) { // {network} is always required a the API level but to // keep backward compat, special "_" network is allowed // that means "no network" r := mux.NewRouter() r.HandleFunc("/v1/{network}/config", bindHandler(handleGetNetworkConfig, ctx, sm)).Methods("GET") r.HandleFunc("/v1/{network}/leases", bindHandler(handleAcquireLease, ctx, sm)).Methods("POST") r.HandleFunc("/v1/{network}/leases/{subnet}", bindHandler(handleRenewLease, ctx, sm)).Methods("PUT") r.HandleFunc("/v1/{network}/leases", bindHandler(handleWatchLeases, ctx, sm)).Methods("GET") r.HandleFunc("/v1/", bindHandler(handleNetworks, ctx, sm)).Methods("GET") l, err := listener(listenAddr, cafile, certfile, keyfile) if err != nil { log.Errorf("Error listening on %v: %v", listenAddr, err) return } c := make(chan error, 1) go func() { c <- http.Serve(l, httpLogger(r)) }() select { case <-ctx.Done(): l.Close() <-c case err := <-c: log.Errorf("Error serving on %v: %v", listenAddr, err) } }
func (n *Network) Init(ctx context.Context, iface *net.Interface, iaddr net.IP, eaddr net.IP) *backend.SubnetDef { var be backend.Backend var sn *backend.SubnetDef steps := []func() error{ func() (err error) { n.Config, err = n.sm.GetNetworkConfig(ctx, n.Name) if err != nil { log.Error("Failed to retrieve network config: ", err) } return }, func() (err error) { be, err = newBackend(n.sm, n.Name, n.Config) if err != nil { log.Error("Failed to create backend: ", err) } else { n.be = be } return }, func() (err error) { sn, err = be.Init(ctx, iface, iaddr, eaddr) if err != nil { log.Errorf("Failed to initialize network %v (type %v): %v", n.Name, n.Config.BackendType, err) } n.lease = sn.Lease return }, func() (err error) { if n.ipMasq { flannelNet := n.Config.Network if err = setupIPMasq(flannelNet); err != nil { log.Errorf("Failed to set up IP Masquerade for network %v: %v", n.Name, err) } } return }, } for _, s := range steps { for ; ; time.Sleep(time.Second) { select { case <-ctx.Done(): return nil default: } err := s() if err == nil { break } } } return sn }
func (rb *HostgwBackend) handleSubnetEvents(batch []subnet.Event) { for _, evt := range batch { switch evt.Type { case subnet.SubnetAdded: log.Infof("Subnet added: %v via %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP) if evt.Lease.Attrs.BackendType != "host-gw" { log.Warningf("Ignoring non-host-gw subnet: type=%v", evt.Lease.Attrs.BackendType) continue } route := netlink.Route{ Dst: evt.Lease.Subnet.ToIPNet(), Gw: evt.Lease.Attrs.PublicIP.ToIP(), LinkIndex: rb.extIface.Index, } if rb.extIaddr.Equal(route.Gw) { continue } if err := netlink.RouteAdd(&route); err != nil { log.Errorf("Error adding route to %v via %v: %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, err) continue } rb.addToRouteList(route) case subnet.SubnetRemoved: log.Info("Subnet removed: ", evt.Lease.Subnet) if evt.Lease.Attrs.BackendType != "host-gw" { log.Warningf("Ignoring non-host-gw subnet: type=%v", evt.Lease.Attrs.BackendType) continue } route := netlink.Route{ Dst: evt.Lease.Subnet.ToIPNet(), Gw: evt.Lease.Attrs.PublicIP.ToIP(), LinkIndex: rb.extIface.Index, } if err := netlink.RouteDel(&route); err != nil { log.Errorf("Error deleting route to %v: %v", evt.Lease.Subnet, err) continue } rb.removeFromRouteList(route) default: log.Error("Internal error: unknown event type: ", int(evt.Type)) } } }
func (be *AwsVpcBackend) checkMatchingRoutes(routeTableID, instanceID, subnet string, ec2c *ec2.EC2) (bool, error) { matchingRouteFound := false filter := newFilter() filter.Add("route.destination-cidr-block", subnet) filter.Add("route.state", "active") input := ec2.DescribeRouteTablesInput{Filters: filter, RouteTableIds: []*string{&routeTableID}} resp, err := ec2c.DescribeRouteTables(&input) if err != nil { return matchingRouteFound, err } for _, routeTable := range resp.RouteTables { for _, route := range routeTable.Routes { if subnet == *route.DestinationCidrBlock && *route.State == "active" { if *route.InstanceId == instanceID { matchingRouteFound = true break } log.Errorf("Deleting invalid *active* matching route: %s, %s \n", *route.DestinationCidrBlock, *route.InstanceId) } } } return matchingRouteFound, nil }
func (rb *HostgwBackend) checkSubnetExistInRoutes() { routeList, err := netlink.RouteList(nil, netlink.FAMILY_V4) if err == nil { for _, route := range rb.rl { exist := false for _, r := range routeList { if r.Dst == nil { continue } if routeEqual(r, route) { exist = true break } } if !exist { if err := netlink.RouteAdd(&route); err != nil { if nerr, ok := err.(net.Error); !ok { log.Errorf("Error recovering route to %v: %v, %v", route.Dst, route.Gw, nerr) } continue } else { log.Infof("Route recovered %v : %v", route.Dst, route.Gw) } } } } }
// WatchLease performs a long term watch of the given network's subnet lease // and communicates addition/deletion events on receiver channel. It takes care // of handling "fall-behind" logic where the history window has advanced too far // and it needs to diff the latest snapshot with its saved state and generate events func WatchLease(ctx context.Context, sm Manager, network string, sn ip.IP4Net, receiver chan Event) { var cursor interface{} for { wr, err := sm.WatchLease(ctx, network, sn, cursor) if err != nil { if err == context.Canceled || err == context.DeadlineExceeded { return } log.Errorf("Subnet watch failed: %v", err) time.Sleep(time.Second) continue } if len(wr.Snapshot) > 0 { receiver <- Event{ Type: EventAdded, Lease: wr.Snapshot[0], } } else { receiver <- wr.Events[0] } cursor = wr.Cursor } }
func (sm *SubnetManager) applySubnetChange(action string, ipn ip.IP4Net, data string) Event { switch action { case "delete", "expire": for i, l := range sm.leases { if l.Network.Equal(ipn) { deleteLease(sm.leases, i) return Event{SubnetRemoved, l} } } log.Errorf("Removed subnet (%s) was not found", ipn) return Event{ SubnetRemoved, SubnetLease{ipn, ""}, } default: for i, l := range sm.leases { if l.Network.Equal(ipn) { sm.leases[i] = SubnetLease{ipn, data} return Event{SubnetAdded, sm.leases[i]} } } sm.leases = append(sm.leases, SubnetLease{ipn, data}) return Event{SubnetAdded, sm.leases[len(sm.leases)-1]} } }
func (m *AwsVpcBackend) checkMatchingRoutes(instanceID, subnet string, ec2c *ec2.EC2) (bool, error) { filter := ec2.NewFilter() filter.Add("route.destination-cidr-block", subnet) filter.Add("route.state", "active") matchingRouteFound := false resp, err := ec2c.DescribeRouteTables([]string{m.cfg.RouteTableID}, filter) if err != nil { return matchingRouteFound, err } for _, routeTable := range resp.RouteTables { for _, route := range routeTable.Routes { if subnet == route.DestinationCidrBlock && route.State == "active" { if route.InstanceId == instanceID { matchingRouteFound = true break } log.Errorf("Deleting invalid *active* matching route: %s, %s \n", route.DestinationCidrBlock, route.InstanceId) } } } return matchingRouteFound, nil }
// WatchLeases performs a long term watch of the given network's subnet leases // and communicates addition/deletion events on receiver channel. It takes care // of handling "fall-behind" logic where the history window has advanced too far // and it needs to diff the latest snapshot with its saved state and generate events func WatchLeases(ctx context.Context, sm Manager, network string, ownLease *Lease, receiver chan []Event) { lw := &leaseWatcher{ ownLease: ownLease, } var cursor interface{} for { res, err := sm.WatchLeases(ctx, network, cursor) if err != nil { if err == context.Canceled || err == context.DeadlineExceeded { return } log.Errorf("Watch subnets: %v", err) time.Sleep(time.Second) continue } cursor = res.Cursor batch := []Event{} if len(res.Events) > 0 { batch = lw.update(res.Events) } else { batch = lw.reset(res.Snapshot) } if batch != nil { receiver <- batch } } }
// WatchNetworks performs a long term watch of flannel networks and communicates // addition/deletion events on receiver channel. It takes care of handling // "fall-behind" logic where the history window has advanced too far and it // needs to diff the latest snapshot with its saved state and generate events func WatchNetworks(ctx context.Context, sm Manager, receiver chan []Event) { nw := newNetWatcher() var cursor interface{} for { res, err := sm.WatchNetworks(ctx, cursor) if err != nil { if err == context.Canceled || err == context.DeadlineExceeded { return } log.Errorf("Watch networks: %v", err) time.Sleep(time.Second) continue } cursor = res.Cursor batch := []Event{} if len(res.Events) > 0 { batch = nw.update(res.Events) } else { batch = nw.reset(res.Snapshot) } if batch != nil { receiver <- batch } } }
func initAndRun(ctx context.Context, sm subnet.Manager, netnames []string) { iface, iaddr, err := lookupIface() if err != nil { log.Error(err) return } if iface.MTU == 0 { log.Errorf("Failed to determine MTU for %s interface", iaddr) return } var eaddr net.IP if len(opts.publicIP) > 0 { eaddr = net.ParseIP(opts.publicIP) } if eaddr == nil { eaddr = iaddr } log.Infof("Using %s as external interface", iaddr) log.Infof("Using %s as external endpoint", eaddr) nets := []*network.Network{} for _, n := range netnames { nets = append(nets, network.New(sm, n, opts.ipMasq)) } wg := sync.WaitGroup{} for _, n := range nets { go func(n *network.Network) { wg.Add(1) defer wg.Done() sn := n.Init(ctx, iface, iaddr, eaddr) if sn != nil { if isMultiNetwork() { path := filepath.Join(opts.subnetDir, n.Name) + ".env" if err := writeSubnetFile(path, n.Config.Network, sn); err != nil { return } } else { if err := writeSubnetFile(opts.subnetFile, n.Config.Network, sn); err != nil { return } daemon.SdNotify("READY=1") } n.Run(ctx) log.Infof("%v exited", n.Name) } }(n) } wg.Wait() }
func (nw *netWatcher) remove(network string) Event { if _, ok := nw.networks[network]; ok { delete(nw.networks, network) } else { log.Errorf("Removed network (%s) was not found", network) } return Event{EventRemoved, Lease{}, network} }
func (lw *leaseWatcher) remove(lease *Lease) Event { for i, l := range lw.leases { if l.Subnet.Equal(lease.Subnet) { lw.leases = deleteLease(lw.leases, i) return Event{EventRemoved, l, ""} } } log.Errorf("Removed subnet (%s) was not found", lease.Subnet) return Event{EventRemoved, *lease, ""} }
func (vb *VXLANBackend) handleL3Miss(miss *netlink.Neigh) { log.Infof("L3 miss: %v", miss.IP) rt := vb.rts.findByNetwork(ip.FromIP(miss.IP)) if rt == nil { log.Infof("Route for %v not found", miss.IP) return } if err := vb.dev.AddL3(neigh{IP: ip.FromIP(miss.IP), MAC: rt.vtepMAC}); err != nil { log.Errorf("AddL3 failed: %v", err) } else { log.Info("AddL3 succeeded") } }
func (n *Network) Run(extIface *backend.ExternalInterface, inited func(bn backend.Network)) { wg := sync.WaitGroup{} For: for { err := n.init() switch err { case nil: break For case context.Canceled: return default: log.Error(err) select { case <-n.ctx.Done(): return case <-time.After(time.Second): } } } inited(n.bn) wg.Add(1) go func() { n.bn.Run(n.ctx) wg.Done() }() wg.Add(1) go func() { subnet.LeaseRenewer(n.ctx, n.sm, n.Name, n.bn.Lease()) wg.Done() }() defer func() { if n.ipMasq { if err := teardownIPMasq(n.Config.Network); err != nil { log.Errorf("Failed to tear down IP Masquerade for network %v: %v", n.Name, err) } } }() wg.Wait() }
func (dev *vxlanDevice) MonitorMisses(misses chan *netlink.Neigh) { nlsock, err := nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_NEIGH) if err != nil { log.Error("Failed to subscribe to netlink RTNLGRP_NEIGH messages") return } for { msgs, err := nlsock.Receive() if err != nil { log.Errorf("Failed to receive from netlink: %v ", err) time.Sleep(1 * time.Second) continue } for _, msg := range msgs { dev.processNeighMsg(msg, misses) } } }
func (sm *SubnetManager) WatchLeases(receiver chan EventBatch, cancel chan bool) { // "catch up" by replaying all the leases we discovered during // AcquireLease var batch EventBatch for _, l := range sm.leases { if !sm.myLease.Network.Equal(l.Network) { batch = append(batch, Event{SubnetAdded, l}) } } if len(batch) > 0 { receiver <- batch } for { resp, err := sm.registry.watchSubnets(sm.lastIndex+1, cancel) // watchSubnets exited by cancel chan being signaled if err == nil && resp == nil { return } var batch *EventBatch if err == nil { batch, err = sm.parseSubnetWatchResponse(resp) } else { batch, err = sm.parseSubnetWatchError(err) } if err != nil { log.Errorf("%v", err) time.Sleep(time.Second) continue } if batch != nil { receiver <- *batch } } }
func (n *Network) runOnce(extIface *backend.ExternalInterface, inited func(bn backend.Network)) error { if err := n.retryInit(); err != nil { return errCanceled } inited(n.bn) ctx, interruptFunc := context.WithCancel(n.ctx) wg := sync.WaitGroup{} wg.Add(1) go func() { n.bn.Run(ctx) wg.Done() }() evts := make(chan subnet.Event) wg.Add(1) go func() { subnet.WatchLease(ctx, n.sm, n.Name, n.bn.Lease().Subnet, evts) wg.Done() }() defer func() { if n.ipMasq { if err := teardownIPMasq(n.Config.Network); err != nil { log.Errorf("Failed to tear down IP Masquerade for network %v: %v", n.Name, err) } } }() defer wg.Wait() dur := n.bn.Lease().Expiration.Sub(time.Now()) - renewMargin for { select { case <-time.After(dur): err := n.sm.RenewLease(n.ctx, n.Name, n.bn.Lease()) if err != nil { log.Error("Error renewing lease (trying again in 1 min): ", err) dur = time.Minute continue } log.Info("Lease renewed, new expiration: ", n.bn.Lease().Expiration) dur = n.bn.Lease().Expiration.Sub(time.Now()) - renewMargin case e := <-evts: switch e.Type { case subnet.EventAdded: n.bn.Lease().Expiration = e.Lease.Expiration dur = n.bn.Lease().Expiration.Sub(time.Now()) - renewMargin case subnet.EventRemoved: log.Warning("Lease has been revoked") interruptFunc() return errInterrupted } case <-n.ctx.Done(): return errCanceled } } }
func (m *AwsVpcBackend) Init(extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (*backend.SubnetDef, error) { // Parse our configuration if len(m.config.Backend) > 0 { if err := json.Unmarshal(m.config.Backend, &m.cfg); err != nil { return nil, fmt.Errorf("error decoding VPC backend config: %v", err) } } // Acquire the lease form subnet manager attrs := subnet.LeaseAttrs{ PublicIP: ip.FromIP(extEaddr), } l, err := m.sm.AcquireLease(m.ctx, m.network, &attrs) switch err { case nil: m.lease = l case context.Canceled, context.DeadlineExceeded: return nil, err default: return nil, fmt.Errorf("failed to acquire lease: %v", err) } // Figure out this machine's EC2 instance ID and region metadataClient := ec2metadata.New(nil) region, err := metadataClient.Region() if err != nil { return nil, fmt.Errorf("error getting EC2 region name: %v", err) } instanceID, err := metadataClient.GetMetadata("instance-id") if err != nil { return nil, fmt.Errorf("error getting EC2 instance ID: %v", err) } ec2c := ec2.New(&aws.Config{Region: aws.String(region)}) if _, err = m.disableSrcDestCheck(instanceID, ec2c); err != nil { log.Infof("Warning- disabling source destination check failed: %v", err) } if m.cfg.RouteTableID == "" { log.Infof("RouteTableID not passed as config parameter, detecting ...") if err := m.detectRouteTableID(instanceID, ec2c); err != nil { return nil, err } } log.Info("RouteRouteTableID: ", m.cfg.RouteTableID) matchingRouteFound, err := m.checkMatchingRoutes(instanceID, l.Subnet.String(), ec2c) if err != nil { log.Errorf("Error describing route tables: %v", err) if ec2Err, ok := err.(awserr.Error); ok { if ec2Err.Code() == "UnauthorizedOperation" { log.Errorf("Note: DescribeRouteTables permission cannot be bound to any resource") } } } if !matchingRouteFound { cidrBlock := l.Subnet.String() deleteRouteInput := &ec2.DeleteRouteInput{RouteTableId: &m.cfg.RouteTableID, DestinationCidrBlock: &cidrBlock} if _, err := ec2c.DeleteRoute(deleteRouteInput); err != nil { if ec2err, ok := err.(awserr.Error); !ok || ec2err.Code() != "InvalidRoute.NotFound" { // an error other than the route not already existing occurred return nil, fmt.Errorf("error deleting existing route for %s: %v", l.Subnet.String(), err) } } // Add the route for this machine's subnet if _, err := m.createRoute(instanceID, l.Subnet.String(), ec2c); err != nil { return nil, fmt.Errorf("unable to add route %s: %v", l.Subnet.String(), err) } } return &backend.SubnetDef{ Net: l.Subnet, MTU: extIface.MTU, }, nil }
func (m *AwsVpcBackend) Init(extIface *net.Interface, extIaddr net.IP, extEaddr net.IP) (*backend.SubnetDef, error) { // Parse our configuration if len(m.config.Backend) > 0 { if err := json.Unmarshal(m.config.Backend, &m.cfg); err != nil { return nil, fmt.Errorf("error decoding VPC backend config: %v", err) } } // Acquire the lease form subnet manager attrs := subnet.LeaseAttrs{ PublicIP: ip.FromIP(extEaddr), } l, err := m.sm.AcquireLease(m.ctx, m.network, &attrs) switch err { case nil: m.lease = l case context.Canceled, context.DeadlineExceeded: return nil, err default: return nil, fmt.Errorf("failed to acquire lease: %v", err) } // Figure out this machine's EC2 instance ID and region identity, err := getInstanceIdentity() if err != nil { return nil, fmt.Errorf("error getting EC2 instance identity: %v", err) } instanceID, ok := identity["instanceId"].(string) if !ok { return nil, fmt.Errorf("invalid EC2 instance ID: %v", identity["instanceId"]) } regionVal, _ := identity["region"].(string) region, ok := aws.Regions[regionVal] if !ok { return nil, fmt.Errorf("invalid AWS region: %v", identity["region"]) } // Setup the EC2 client auth, err := aws.GetAuth("", "") if err != nil { return nil, fmt.Errorf("error getting AWS credentials from environment: %v", err) } ec2c := ec2.New(auth, region) if _, err = m.disableSrcDestCheck(instanceID, ec2c); err != nil { log.Infof("Warning- disabling source destination check failed: %v", err) } if m.cfg.RouteTableID == "" { log.Infof("RouteTableID not passed as config parameter, detecting ...") if err := m.detectRouteTableID(instanceID, ec2c); err != nil { return nil, err } } log.Info("RouteRouteTableID: ", m.cfg.RouteTableID) matchingRouteFound, err := m.checkMatchingRoutes(instanceID, l.Subnet.String(), ec2c) if err != nil { log.Errorf("Error describing route tables: %v", err) if ec2Err, ok := err.(*ec2.Error); ok { if ec2Err.Code == "UnauthorizedOperation" { log.Errorf("Note: DescribeRouteTables permission cannot be bound to any resource") } } } if !matchingRouteFound { if _, err := ec2c.DeleteRoute(m.cfg.RouteTableID, l.Subnet.String()); err != nil { if ec2err, ok := err.(*ec2.Error); !ok || ec2err.Code != "InvalidRoute.NotFound" { // an error other than the route not already existing occurred return nil, fmt.Errorf("error deleting existing route for %s: %v", l.Subnet.String(), err) } } // Add the route for this machine's subnet if _, err := m.createRoute(instanceID, l.Subnet.String(), ec2c); err != nil { return nil, fmt.Errorf("unable to add route %s: %v", l.Subnet.String(), err) } } return &backend.SubnetDef{ Net: l.Subnet, MTU: extIface.MTU, }, nil }
func (m *AwsVpcBackend) Init(extIface *net.Interface, extIP net.IP) (*backend.SubnetDef, error) { // Parse our configuration if len(m.config.Backend) > 0 { if err := json.Unmarshal(m.config.Backend, &m.cfg); err != nil { return nil, fmt.Errorf("error decoding VPC backend config: %v", err) } } // Acquire the lease form subnet manager attrs := subnet.LeaseAttrs{ PublicIP: ip.FromIP(extIP), } l, err := m.sm.AcquireLease(m.ctx, m.network, &attrs) switch err { case nil: m.lease = l case context.Canceled, context.DeadlineExceeded: return nil, err default: return nil, fmt.Errorf("failed to acquire lease: %v", err) } // Figure out this machine's EC2 instance ID and region identity, err := getInstanceIdentity() if err != nil { return nil, fmt.Errorf("error getting EC2 instance identity: %v", err) } instanceID, ok := identity["instanceId"].(string) if !ok { return nil, fmt.Errorf("invalid EC2 instance ID: %v", identity["instanceId"]) } regionVal, _ := identity["region"].(string) region, ok := aws.Regions[regionVal] if !ok { return nil, fmt.Errorf("invalid AWS region: %v", identity["region"]) } // Setup the EC2 client auth, err := aws.GetAuth("", "") if err != nil { return nil, fmt.Errorf("error getting AWS credentials from environment: %v", err) } ec2c := ec2.New(auth, region) if m.cfg.RouteTableID == "" { log.Infof("RouteTableID not passed as config parameter, attempting to detect") routeTableID, err := m.DetectRouteTableID(instanceID, ec2c) if err != nil { return nil, err } log.Info("Detected routeRouteTableID: ", routeTableID) m.cfg.RouteTableID = routeTableID } filter := ec2.NewFilter() filter.Add("route.destination-cidr-block", l.Subnet.String()) filter.Add("route.state", "active") resp, err := ec2c.DescribeRouteTables([]string{m.cfg.RouteTableID}, filter) if err != nil { log.Errorf("Error describing route tables: %v", err) if ec2Err, ok := err.(*ec2.Error); ok { if ec2Err.Code == "UnauthorizedOperation" { log.Errorf("Note: describeRouteTables permission cannot be bound to any resource") } } } else { for _, routeTable := range resp.RouteTables { for _, route := range routeTable.Routes { if l.Subnet.String() == route.DestinationCidrBlock && route.State == "active" { log.Errorf("Matching *active* entry to: %s that will be deleted: %s, %s \n", l.Subnet.String(), route.DestinationCidrBlock, route.GatewayId) } } } } // Delete route for this machine's subnet if it already exists if _, err := ec2c.DeleteRoute(m.cfg.RouteTableID, l.Subnet.String()); err != nil { if ec2err, ok := err.(*ec2.Error); !ok || ec2err.Code != "InvalidRoute.NotFound" { // an error other than the route not already existing occurred return nil, fmt.Errorf("error deleting existing route for %s: %v", l.Subnet.String(), err) } } // Add the route for this machine's subnet route := &ec2.CreateRoute{ RouteTableId: m.cfg.RouteTableID, InstanceId: instanceID, DestinationCidrBlock: l.Subnet.String(), } if _, err := ec2c.CreateRoute(route); err != nil { return nil, fmt.Errorf("unable to add route %+v: %v", route, err) } return &backend.SubnetDef{ Net: l.Subnet, MTU: extIface.MTU, }, nil }