// delRouteIface clean up the required L3 mode default ns route
func delRouteIface(ipVlanL3Network *net.IPNet, iface netlink.Link) error {
return netlink.RouteDel(&netlink.Route{
LinkIndex: iface.Attrs().Index,
Scope: netlink.SCOPE_LINK,
Dst: ipVlanL3Network,
})
}
// delRemoteRoute deletes a host-scoped route to a device.
func delRemoteRoute(neighborNetwork *net.IPNet, nextHop net.IP, iface netlink.Link) error {
return netlink.RouteDel(&netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
LinkIndex: iface.Attrs().Index,
Dst: neighborNetwork,
Gw: nextHop,
})
}
// delRoute deletes any netlink route
func delRoute(route netlink.Route, iface netlink.Link) error {
return netlink.RouteDel(&netlink.Route{
Scope: route.Scope,
LinkIndex: iface.Attrs().Index,
Dst: route.Dst,
Gw: route.Gw,
})
}
func (dev *vxlanDevice) DelRoute(subnet ip.IP4Net) error {
route := &netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
Dst: subnet.ToIPNet(),
Gw: subnet.IP.ToIP(),
}
log.Infof("calling RouteDel: %s", subnet)
return netlink.RouteDel(route)
}
// delRemoteRoute deletes a host-scoped route to a device.
func delRemoteRoute(neighborNetwork *net.IPNet, nextHop net.IP, iface netlink.Link) error {
log.Debugf("Deleting route: IP Prefix: [ %s ] - Next Hop: [ %s ] - Source Interface: [ %s ]", neighborNetwork, nextHop, iface.Attrs().Name)
return netlink.RouteDel(&netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
LinkIndex: iface.Attrs().Index,
Dst: neighborNetwork,
Gw: nextHop,
})
}
// delRoute deletes any netlink route
func delRoute(route netlink.Route, iface netlink.Link) error {
log.Debugf("Deleting route in the default namespace for IPVlan L3 mode with the following:")
log.Debugf("IP Prefix: [ %s ] - Next Hop: [ %s ] - Source Interface: [ %s ]", route.Dst, route.Gw, iface.Attrs().Name)
return netlink.RouteDel(&netlink.Route{
Scope: route.Scope,
LinkIndex: iface.Attrs().Index,
Dst: route.Dst,
Gw: route.Gw,
})
}
func (t *AWSVPCTracker) deleteHostRoute(cidr string) error {
dst, err := parseCIDR(cidr)
if err != nil {
return err
}
route := &netlink.Route{
LinkIndex: t.linkIndex,
Dst: dst,
Scope: netlink.SCOPE_LINK,
}
return netlink.RouteDel(route)
}
//delete default routes
//FIXME all default routes will be erased
func delDefaultRoute() error {
routes, _ := netlink.RouteList(nil, netlink.FAMILY_V4)
for _, route := range routes {
if route.Dst != nil || route.Src != nil {
continue
}
if err := netlink.RouteDel(&route); err != nil {
return err
}
}
return nil
}
// AddHostRoute adds a route to a device using netlink into the Linux default namespace.
func delNetlinkRoute(neighborNetwork *net.IPNet, nextHop net.IP, netIface string) error {
iface, err := netlink.LinkByName(netIface)
if err != nil {
return err
}
log.Infof("IP Prefix: [ %s ] - Next Hop: [ %s ] - Source Interface: [ %s ]", neighborNetwork, nextHop, iface.Attrs().Name)
return netlink.RouteDel(&netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
LinkIndex: iface.Attrs().Index,
Dst: neighborNetwork,
Gw: nextHop,
})
}
func removeAllRoutesOnLink(link netlink.Link) error {
routes, err := netlink.RouteList(link, netlink.FAMILY_V4)
if err != nil {
return errwrap.Wrap(fmt.Errorf("cannot list routes on link %q", link.Attrs().Name), err)
}
for _, route := range routes {
if err := netlink.RouteDel(&route); err != nil {
return errwrap.Wrap(fmt.Errorf("error in time of route removal for route %q", route), err)
}
}
return nil
}
// Delete a route from the namespace routing table.
func removeRoute(path string, dest *net.IPNet, nh net.IP) error {
return nsInvoke(path, func(nsFD int) error { return nil }, func(callerFD int) error {
gwRoutes, err := netlink.RouteGet(nh)
if err != nil {
return fmt.Errorf("route for the next hop could not be found: %v", err)
}
return netlink.RouteDel(&netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
LinkIndex: gwRoutes[0].LinkIndex,
Gw: gwRoutes[0].Gw,
Dst: dest,
})
})
}
func delAllRoutesVia(s net.Addr) error {
src := s.(*net.TCPAddr).IP
routes, err := netlink.RouteList(nil, netlink.FAMILY_ALL)
if err != nil {
log.Error("Failed to get routes")
log.Error(err)
return err
}
for _, r := range routes {
if r.Gw.Equal(src) {
netlink.RouteDel(&r)
}
}
return nil
}
func (rb *HostgwBackend) handleSubnetEvents(batch subnet.EventBatch) {
for _, evt := range batch {
switch evt.Type {
case subnet.SubnetAdded:
log.Infof("Subnet added: %v via %v", evt.Lease.Network, evt.Lease.Attrs.PublicIP)
if evt.Lease.Attrs.BackendType != "host-gw" {
log.Warningf("Ignoring non-host-gw subnet: type=%v", evt.Lease.Attrs.BackendType)
continue
}
route := netlink.Route{
Dst: evt.Lease.Network.ToIPNet(),
Gw: evt.Lease.Attrs.PublicIP.ToIP(),
LinkIndex: rb.extIface.Index,
}
if err := netlink.RouteAdd(&route); err != nil {
log.Errorf("Error adding route to %v via %v: %v", evt.Lease.Network, evt.Lease.Attrs.PublicIP, err)
continue
}
case subnet.SubnetRemoved:
log.Info("Subnet removed: ", evt.Lease.Network)
if evt.Lease.Attrs.BackendType != "host-gw" {
log.Warningf("Ignoring non-host-gw subnet: type=%v", evt.Lease.Attrs.BackendType)
continue
}
route := netlink.Route{
Dst: evt.Lease.Network.ToIPNet(),
Gw: evt.Lease.Attrs.PublicIP.ToIP(),
LinkIndex: rb.extIface.Index,
}
if err := netlink.RouteDel(&route); err != nil {
log.Errorf("Error deleting route to %v: %v", evt.Lease.Network, err)
continue
}
default:
log.Error("Internal error: unknown event type: ", int(evt.Type))
}
}
}
func routingDown() {
veth, err := netlink.LinkByName("myveth1")
if err != nil {
panic(err)
}
routes := createRoutes(veth)
for _, route := range routes {
fmt.Println("Removing route", route)
err := netlink.RouteDel(route)
if err != nil {
// panic(err)
fmt.Println(err)
}
}
err = netlink.LinkSetDown(veth)
if err != nil {
panic(err)
}
}
func programGateway(path string, gw net.IP, isAdd bool) error {
return nsInvoke(path, func(nsFD int) error { return nil }, func(callerFD int) error {
gwRoutes, err := netlink.RouteGet(gw)
if err != nil {
return fmt.Errorf("route for the gateway could not be found: %v", err)
}
if isAdd {
return netlink.RouteAdd(&netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
LinkIndex: gwRoutes[0].LinkIndex,
Gw: gw,
})
}
return netlink.RouteDel(&netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
LinkIndex: gwRoutes[0].LinkIndex,
Gw: gw,
})
})
}
func processRoute(ru *exportRoute, s net.Addr) error {
src := s.(*net.TCPAddr).IP
_, err := netlink.AddrList(nil, netlink.FAMILY_ALL)
if err != nil {
log.Error("Failed to get addresses")
log.Error(err)
return err
}
switch {
case ru.Type == syscall.RTM_NEWROUTE:
r := &netlink.Route{
Dst: ru.Dst,
Gw: src,
Priority: ru.Priority + 100,
}
err := netlink.RouteAdd(r)
if err != nil {
log.Errorf("Failed to add route: %v", r)
log.Error(err)
return err
}
case ru.Type == syscall.RTM_DELROUTE:
routes, err := netlink.RouteGet(ru.Dst.IP)
if err != nil {
log.Error("Failed to get routes")
log.Error(err)
return err
}
for _, r := range routes {
if r.Gw.Equal(src) {
netlink.RouteDel(&r)
}
}
}
return nil
}
func RouteDel(cResp chan<- *Response, rawArgs *json.RawMessage, tag string) {
args := &struct {
Ifname string `json:"ifname"`
Dst string `json:"dst"`
Src string `json:"src"`
Gw string `json:"gateway"`
Scope netlink.Scope `json:"scope"`
}{}
json.Unmarshal(*rawArgs, &args)
route, err := NewNetlinkRoute(args.Ifname, args.Dst, args.Src, args.Gw, args.Scope)
if err != nil {
cResp <- &Response{nil, tag, NewRTNetlinkError(err)}
return
}
if err := netlink.RouteDel(route); err != nil {
cResp <- &Response{nil, tag, NewRTNetlinkError(err)}
return
}
cResp <- &Response{true, tag, nil}
}
func (d *driver) deleteNetwork(w http.ResponseWriter, r *http.Request) {
var req deleteNetworkRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
panic(err)
}
if req.NetworkID != d.networkID {
panic(fmt.Errorf("no network with ID %s", req.NetworkID))
}
route := &netlink.Route{
LinkIndex: d.hostLink.Attrs().Index,
Scope: netlink.SCOPE_LINK,
Dst: d.ipnet,
}
if err := netlink.RouteDel(route); err != nil {
Warn.Printf("RouteDel failed: %s\n", err.Error())
}
d.networkID = ""
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{}`)
}
func (n *network) handleSubnetEvents(batch []subnet.Event) {
for _, evt := range batch {
switch evt.Type {
case subnet.EventAdded:
log.Infof("Subnet added: %v via %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP)
if evt.Lease.Attrs.BackendType != "host-gw" {
log.Warningf("Ignoring non-host-gw subnet: type=%v", evt.Lease.Attrs.BackendType)
continue
}
route := netlink.Route{
Dst: evt.Lease.Subnet.ToIPNet(),
Gw: evt.Lease.Attrs.PublicIP.ToIP(),
LinkIndex: n.linkIndex,
}
// Check if route exists before attempting to add it
routeList, err := netlink.RouteListFiltered(netlink.FAMILY_V4, &netlink.Route{
Dst: route.Dst,
}, netlink.RT_FILTER_DST)
if err != nil {
log.Warningf("Unable to list routes: %v", err)
}
// Check match on Dst for match on Gw
if len(routeList) > 0 && !routeList[0].Gw.Equal(route.Gw) {
// Same Dst different Gw. Remove it, correct route will be added below.
log.Warningf("Replacing existing route to %v via %v with %v via %v.", evt.Lease.Subnet, routeList[0].Gw, evt.Lease.Subnet, evt.Lease.Attrs.PublicIP)
if err := netlink.RouteDel(&route); err != nil {
log.Errorf("Error deleting route to %v: %v", evt.Lease.Subnet, err)
continue
}
}
if len(routeList) > 0 && routeList[0].Gw.Equal(route.Gw) {
// Same Dst and same Gw, keep it and do not attempt to add it.
log.Infof("Route to %v via %v already exists, skipping.", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP)
} else if err := netlink.RouteAdd(&route); err != nil {
log.Errorf("Error adding route to %v via %v: %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, err)
continue
}
n.addToRouteList(route)
case subnet.EventRemoved:
log.Info("Subnet removed: ", evt.Lease.Subnet)
if evt.Lease.Attrs.BackendType != "host-gw" {
log.Warningf("Ignoring non-host-gw subnet: type=%v", evt.Lease.Attrs.BackendType)
continue
}
route := netlink.Route{
Dst: evt.Lease.Subnet.ToIPNet(),
Gw: evt.Lease.Attrs.PublicIP.ToIP(),
LinkIndex: n.linkIndex,
}
if err := netlink.RouteDel(&route); err != nil {
log.Errorf("Error deleting route to %v: %v", evt.Lease.Subnet, err)
continue
}
n.removeFromRouteList(route)
default:
log.Error("Internal error: unknown event type: ", int(evt.Type))
}
}
}
func AddDHCPNetwork() {
if ok := utils.ValidateHostIface(flat.CliIF); !ok {
log.Fatalf("the host-interface [ %s ] was not found.", flat.CliIF)
}
hostmacvlanname, _ := utils.GenerateRandomName(hostprefix, hostlen)
hostEth, _ := netlink.LinkByName(flat.CliIF)
//create the macvlan device
macvlandev := &netlink.Macvlan{
LinkAttrs: netlink.LinkAttrs{
Name: hostmacvlanname,
ParentIndex: hostEth.Attrs().Index,
},
Mode: netlink.MACVLAN_MODE_BRIDGE,
}
if err := netlink.LinkAdd(macvlandev); err != nil {
log.Fatalf("failed to create Macvlan: [ %v ] with the error: %s", macvlandev.Attrs().Name, err)
}
dockerPid := utils.DockerPid(flat.CliCName)
netlink.LinkSetNsPid(macvlandev, dockerPid)
runtime.LockOSThread()
defer runtime.UnlockOSThread()
//get root network namespace
origns, _ := netns.Get()
defer origns.Close()
//enter the docker container network
dockerNS, _ := netns.GetFromPid(dockerPid)
defer dockerNS.Close()
//get the dochclient name
dhcpClientPath := "/home/fmzhen/go/src/github.com/fmzhen/docker-macvlan/macvlan/dhcp/dhcpclient.sh"
out, err := exec.Command(dhcpClientPath).Output()
if err != nil {
log.Fatal("exec the dhcpclient.sh error: ", err)
}
dhcpClient := string(out)
// like ip netns exec xxx, just the netns ,so the command in host can exec
netns.Set(dockerNS)
// use macvlandev can cause error,need type assertion. netlink.Macvlan not must be netlink.Link,fmz
macvlandev1, _ := netlink.LinkByName(macvlandev.Attrs().Name)
netlink.LinkSetDown(macvlandev1)
netlink.LinkSetName(macvlandev1, "eth1")
netlink.LinkSetUp(macvlandev1)
//delete the default route
routes, _ := netlink.RouteList(nil, netlink.FAMILY_V4)
for _, r := range routes {
if r.Dst == nil {
if err := netlink.RouteDel(&r); err != nil {
log.Warnf("delete the default error: ", err)
}
}
}
//it doesn't work, the problem at the atgs don't pass to the shell script
dhcpReqpath := "/home/fmzhen/go/src/github.com/fmzhen/docker-macvlan/macvlan/dhcp/dhcpReq.sh"
exec.Command(dhcpReqpath, dhcpClient, string(dockerPid), flat.CliCName).Run()
netns.Set(origns)
}
func setupContainerVeth(netns, ifName string, mtu int, pr *types.Result) (string, error) {
// The IPAM result will be something like IP=192.168.3.5/24, GW=192.168.3.1.
// What we want is really a point-to-point link but veth does not support IFF_POINTOPONT.
// Next best thing would be to let it ARP but set interface to 192.168.3.5/32 and
// add a route like "192.168.3.0/24 via 192.168.3.1 dev $ifName".
// Unfortunately that won't work as the GW will be outside the interface's subnet.
// Our solution is to configure the interface with 192.168.3.5/24, then delete the
// "192.168.3.0/24 dev $ifName" route that was automatically added. Then we add
// "192.168.3.1/32 dev $ifName" and "192.168.3.0/24 via 192.168.3.1 dev $ifName".
// In other words we force all traffic to ARP via the gateway except for GW itself.
var hostVethName string
err := ns.WithNetNSPath(netns, false, func(hostNS *os.File) error {
hostVeth, _, err := ip.SetupVeth(ifName, mtu, hostNS)
if err != nil {
return err
}
if err = ipam.ConfigureIface(ifName, pr); err != nil {
return err
}
contVeth, err := netlink.LinkByName(ifName)
if err != nil {
return fmt.Errorf("failed to look up %q: %v", ifName, err)
}
// Delete the route that was automatically added
route := netlink.Route{
LinkIndex: contVeth.Attrs().Index,
Dst: &net.IPNet{
IP: pr.IP4.IP.IP.Mask(pr.IP4.IP.Mask),
Mask: pr.IP4.IP.Mask,
},
Scope: netlink.SCOPE_NOWHERE,
}
if err := netlink.RouteDel(&route); err != nil {
return fmt.Errorf("failed to delete route %v: %v", route, err)
}
for _, r := range []netlink.Route{
netlink.Route{
LinkIndex: contVeth.Attrs().Index,
Dst: &net.IPNet{
IP: pr.IP4.Gateway,
Mask: net.CIDRMask(32, 32),
},
Scope: netlink.SCOPE_LINK,
Src: pr.IP4.IP.IP,
},
netlink.Route{
LinkIndex: contVeth.Attrs().Index,
Dst: &net.IPNet{
IP: pr.IP4.IP.IP.Mask(pr.IP4.IP.Mask),
Mask: pr.IP4.IP.Mask,
},
Scope: netlink.SCOPE_UNIVERSE,
Gw: pr.IP4.Gateway,
Src: pr.IP4.IP.IP,
},
} {
if err := netlink.RouteAdd(&r); err != nil {
return fmt.Errorf("failed to add route %v: %v", r, err)
}
}
hostVethName = hostVeth.Attrs().Name
return nil
})
return hostVethName, err
}
func DeleteDefaultGateway(route *netlink.Route) error {
return netlink.RouteDel(route)
}
func destroyTunnel(dst net.IP) (net.IP, error) {
// Determine the src and dst ips for the tunnel
key := dst.String()
tunnel := getTunnel(dst.String())
if tunnel == nil {
s := fmt.Sprintf("Failed to find tunnel to dst %s", dst)
glog.Errorf(s)
return nil, fmt.Errorf(s)
}
src := opts.src
srcNet := netlink.NewIPNet(tunnel.Src)
dstNet := netlink.NewIPNet(tunnel.Dst)
glog.Infof("Destroying Tunnel: %v, %v", tunnel.Src, tunnel.Dst)
for _, state := range getStates(tunnel.Reqid, src, dst, 0, 0, nil, nil) {
// crate xfrm state rules
err := netlink.XfrmStateDel(&state)
if err != nil {
glog.Errorf("Failed to delete state %v: %v", state, err)
}
}
for _, policy := range getPolicies(tunnel.Reqid, src, dst, srcNet, dstNet) {
// create xfrm policy rules
err := netlink.XfrmPolicyDel(&policy)
if err != nil {
glog.Errorf("Failed to delete policy %v: %v", policy, err)
}
}
index, err := getLinkIndex(src)
if err != nil {
glog.Errorf("Failed to get link for address: %v", err)
} else {
// del source route to tunnel ips device
route := &netlink.Route{
Scope: netlink.SCOPE_LINK,
Src: tunnel.Src,
Dst: dstNet,
LinkIndex: index,
}
err = netlink.RouteDel(route)
if err != nil {
glog.Errorf("Failed to delete route %v: %v", route, err)
}
}
// del IP address to loopback device
lo, err := netlink.LinkByName("lo")
if err != nil {
glog.Errorf("Failed to get loopback device: %v", err)
} else {
err = netlink.AddrDel(lo, &netlink.Addr{IPNet: srcNet})
if err != nil {
glog.Errorf("Failed to delete %v from loopback: %v", tunnel.Src, err)
}
}
if tunnel.SrcPort != 0 {
deleteEncapListener(getListener(key))
releasePort(tunnel.SrcPort)
}
unreserveIP(tunnel.Src)
unreserveIP(tunnel.Dst)
removeTunnel(key)
glog.Infof("Finished destroying tunnel: %v, %v", tunnel.Src, tunnel.Dst)
return opts.external, nil
}
//netlink is not avaible in MAC OS, build fail.
func AddContainerNetworking() {
if CliIF == "" {
log.Fatal("the host-interface is missing,please give one")
}
if ok := utils.ValidateHostIface(CliIF); !ok {
log.Fatalf("the host-interface [ %s ] was not found.", CliIF)
}
hostmacvlanname, _ := utils.GenerateRandomName(hostprefix, hostlen)
hostEth, _ := netlink.LinkByName(CliIF)
//create the macvlan device
macvlandev := &netlink.Macvlan{
LinkAttrs: netlink.LinkAttrs{
Name: hostmacvlanname,
ParentIndex: hostEth.Attrs().Index,
},
Mode: netlink.MACVLAN_MODE_BRIDGE,
}
if err := netlink.LinkAdd(macvlandev); err != nil {
log.Fatalf("failed to create Macvlan: [ %v ] with the error: %s", macvlandev.Attrs().Name, err)
}
// log.Infof("Created Macvlan port: [ %s ] using the mode: [ %s ]", macvlan.Name, macvlanMode)
// ugly, actually ,can get the ns from netns.getfromDocker. the netns have many function, netns.getformpid
// netns.getfromdocker the arg can not be the container name
dockerPid := utils.DockerPid(CliCName)
//the macvlandev can be use directly, don't get netlink.byname again.
netlink.LinkSetNsPid(macvlandev, dockerPid)
runtime.LockOSThread()
defer runtime.UnlockOSThread()
//get root network namespace
origns, _ := netns.Get()
defer origns.Close()
//enter the docker container network
dockerNS, _ := netns.GetFromPid(dockerPid)
defer dockerNS.Close()
netns.Set(dockerNS)
// use macvlandev can cause error,need type assertion. netlink.Macvlan not must be netlink.Link,fmz(realy? the vlan action add)
// it is wrong,
macvlandev1, _ := netlink.LinkByName(macvlandev.Attrs().Name)
// when the eth is up, set name fail,: Device or resource busy
netlink.LinkSetDown(macvlandev1)
netlink.LinkSetName(macvlandev1, "eth1")
addr, err := netlink.ParseAddr(CliIP)
if err != nil {
log.Fatalf("failed to parse the ip address %v", CliIP)
}
netlink.AddrAdd(macvlandev1, addr)
netlink.LinkSetUp(macvlandev1)
/* set the default route, have some problem. Dst == 0.0.0.0/0? no
defaultgw := &netlink.Route{
Dst: nil,
}
netlink.RouteDel(defaultgw)
ip, _ := net.ParseIP("8.8.8.8")
routes, _ := netlink.RouteGet(ip)
for _, r := range routes {
netlink.RouteDel(&r)
}
*/
//if use ip instruction, it also can config the container, --privileged have no effect.
// The sublime test code(test this function) is strange, it only can avaiable in first time. And then fail(even need to reboot)
// got it,
//following code successfully delete the default route in docker container,but error in my host ,no such process
routes, _ := netlink.RouteList(nil, netlink.FAMILY_V4)
for _, r := range routes {
if r.Dst == nil {
if err := netlink.RouteDel(&r); err != nil {
log.Warnf("delete the default error: ", err)
}
}
}
if CligwIP == "" {
log.Fatal("container gw is null")
}
defaultRoute := &netlink.Route{
Dst: nil,
Gw: net.ParseIP(CligwIP),
LinkIndex: macvlandev1.Attrs().Index,
}
if err := netlink.RouteAdd(defaultRoute); err != nil {
log.Warnf("create default route error: ", err)
}
netns.Set(origns)
}
func (t *BaseOperations) RouteDel(route *netlink.Route) error {
return netlink.RouteDel(route)
}