func IpAddrDel(cResp chan<- *Response, rawArgs *json.RawMessage, tag string) {
args := &struct {
Ifname string `json:"ifname"`
IpCidr string `json:"ip"`
}{}
json.Unmarshal(*rawArgs, &args)
iface, err := netlink.LinkByName(args.Ifname)
if err != nil {
cResp <- &Response{nil, tag, NewRTNetlinkError(err)}
return
}
ip, err := netlink.ParseAddr(args.IpCidr)
if err != nil {
cResp <- &Response{nil, tag, NewRTNetlinkError(err)}
return
}
if err := netlink.AddrDel(iface, ip); err != nil {
cResp <- &Response{nil, tag, NewRTNetlinkError(err)}
return
}
cResp <- &Response{true, tag, nil}
}
func main() {
lo, _ := netlink.LinkByName("lo")
fmt.Println(lo)
addr, _ := netlink.ParseAddr("127.0.0.2/8")
fmt.Println(addr)
netlink.AddrAdd(lo, addr)
fmt.Println(netlink.AddrList(lo, netlink.FAMILY_ALL))
req := nl.NewNetlinkRequest(syscall.NLMSG_MIN_TYPE, syscall.NLM_F_REQUEST)
data := append([]byte("TASKSTATS"), 0)
a := &Attr{
Type: 2,
Data: data,
}
Hdr := []byte{
3,
0,
0,
0,
}
m := &Message{
Type: 16,
Pid: -1,
Seq: -1,
Flags: NLM_F_REQUEST,
Payload: append(Hdr, a.Dump()...),
}
req.AddData(m)
res, _ := req.Execute(syscall.NETLINK_GENERIC, 0)
fmt.Println(res)
fmt.Println(parse_attributes(res[0][4:])[1])
}
func applyNetConf(link netlink.Link, netConf config.InterfaceConfig) error {
if netConf.DHCP {
log.Infof("Running DHCP on %s", link.Attrs().Name)
cmd := exec.Command("dhcpcd", "-A4", "-e", "force_hostname=true", link.Attrs().Name)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
log.Error(err)
}
} else if netConf.IPV4LL {
if err := AssignLinkLocalIP(link); err != nil {
log.Error("IPV4LL set failed")
return err
}
} else if netConf.Address == "" {
return nil
} else {
addr, err := netlink.ParseAddr(netConf.Address)
if err != nil {
return err
}
if err := netlink.AddrAdd(link, addr); err != nil {
log.Error("addr add failed")
return err
}
log.Infof("Set %s on %s", netConf.Address, link.Attrs().Name)
}
if netConf.MTU > 0 {
if err := netlink.LinkSetMTU(link, netConf.MTU); err != nil {
log.Error("set MTU Failed")
return err
}
}
if err := netlink.LinkSetUp(link); err != nil {
log.Error("failed to setup link")
return err
}
if netConf.Gateway != "" {
gatewayIp := net.ParseIP(netConf.Gateway)
if gatewayIp == nil {
return errors.New("Invalid gateway address " + netConf.Gateway)
}
route := netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
Gw: net.ParseIP(netConf.Gateway),
}
if err := netlink.RouteAdd(&route); err != nil {
log.Error("gateway set failed")
return err
}
log.Infof("Set default gateway %s", netConf.Gateway)
}
return nil
}
/* SetupNetwork()
*/
func SetupNetwork(addr string) error {
// Bring up loop back interface.
lo, e := netlink.LinkByName("lo")
if e != nil {
return fmt.Errorf("Failed to find loopback interface: %v", e)
}
if e := netlink.LinkSetUp(lo); e != nil {
return fmt.Errorf("Failed to setup loopback interface: %v", e)
}
if len(addr) > 0 {
veth, e := netlink.LinkByName("veth0")
if e != nil {
return fmt.Errorf("Failed to find veth interface: %v", e)
}
addr, e := netlink.ParseAddr(addr)
if e != nil {
return fmt.Errorf("Failed to parse NetworkAddr: %v", e)
}
netlink.AddrAdd(veth, addr)
if e := netlink.LinkSetUp(veth); e != nil {
return fmt.Errorf("Network link failed to come up: %v", e)
}
}
return nil
}
func routingUp() {
veth, err := netlink.LinkByName("myveth1")
if err != nil {
panic(err)
}
err = netlink.LinkSetUp(veth)
if err != nil {
panic(err)
}
addr, _ := netlink.ParseAddr("172.19.80.2/24")
err = netlink.AddrAdd(veth, addr)
if err != nil {
panic(err)
}
routes := createRoutes(veth)
for _, route := range routes {
fmt.Println("Adding route", route)
err := netlink.RouteAdd(route)
if err != nil {
fmt.Println(err)
// panic(err)
}
}
}
func addrAdd(t *testing.T, l netlink.Link, a string) {
addr, err := netlink.ParseAddr(a)
if err != nil {
t.Fatal(err)
}
err = netlink.AddrAdd(l, addr)
if err != nil && err != syscall.EEXIST {
t.Fatal(err)
}
}
func applyNetConf(link netlink.Link, netConf InterfaceConfig) error {
if netConf.IPV4LL {
if err := AssignLinkLocalIP(link); err != nil {
log.Errorf("IPV4LL set failed: %v", err)
return err
}
} else if netConf.Address == "" {
return nil
} else {
addr, err := netlink.ParseAddr(netConf.Address)
if err != nil {
return err
}
if err := netlink.AddrAdd(link, addr); err != nil {
//Ignore this error
log.Errorf("addr add failed: %v", err)
} else {
log.Infof("Set %s on %s", netConf.Address, link.Attrs().Name)
}
}
if netConf.MTU > 0 {
if err := netlink.LinkSetMTU(link, netConf.MTU); err != nil {
log.Errorf("set MTU Failed: %v", err)
return err
}
}
if err := netlink.LinkSetUp(link); err != nil {
log.Errorf("failed to setup link: %v", err)
return err
}
if netConf.Gateway != "" {
gatewayIp := net.ParseIP(netConf.Gateway)
if gatewayIp == nil {
return errors.New("Invalid gateway address " + netConf.Gateway)
}
route := netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
Gw: net.ParseIP(netConf.Gateway),
}
if err := netlink.RouteAdd(&route); err != nil {
log.Errorf("gateway set failed: %v", err)
return err
}
log.Infof("Set default gateway %s", netConf.Gateway)
}
return nil
}
func TestPortInformation(t *testing.T) {
mockContainerInfo := &plmodels.ContainerInfo{}
mockContainerConfig := &plmodels.ContainerConfig{}
containerID := "foo"
mockContainerConfig.ContainerID = &containerID
mockHostConfig := &container.HostConfig{}
portMap := nat.PortMap{}
port, _ := nat.NewPort("tcp", "80")
portBinding := nat.PortBinding{
HostIP: "127.0.0.1",
HostPort: "8000",
}
portBindings := []nat.PortBinding{portBinding}
portMap[port] = portBindings
mockHostConfig.PortBindings = portMap
mockContainerInfo.ContainerConfig = mockContainerConfig
ip, _ := netlink.ParseAddr("192.168.1.1/24")
ips := []netlink.Addr{*ip}
co := viccontainer.NewVicContainer()
co.HostConfig = mockHostConfig
co.ContainerID = containerID
co.Name = "bar"
cache.ContainerCache().AddContainer(co)
ports := portInformation(mockContainerInfo, ips)
assert.NotEmpty(t, ports, "There should be bound IPs")
assert.Equal(t, len(ports), 1, "Expected 1 port binding, found %d", len(ports))
port, _ = nat.NewPort("tcp", "80")
portBinding = nat.PortBinding{
HostIP: "127.0.0.1",
HostPort: "00",
}
portMap[port] = portBindings
ports = portInformation(mockContainerInfo, ips)
assert.NotEmpty(t, ports, "There should be 1 bound IP")
assert.Equal(t, len(ports), 1, "Expected 1 port binding, found %d", len(ports))
port, _ = nat.NewPort("tcp", "800")
portBinding = nat.PortBinding{
HostIP: "127.0.0.1",
HostPort: "800",
}
portMap[port] = portBindings
ports = portInformation(mockContainerInfo, ips)
assert.Equal(t, len(ports), 2, "Expected 2 port binding, found %d", len(ports))
}
// SetInterfaceIP : Set IP address of an interface
func SetInterfaceIP(name string, ipstr string) error {
iface, err := netlink.LinkByName(name)
if err != nil {
return err
}
ipaddr, err := netlink.ParseAddr(ipstr)
if err != nil {
return err
}
netlink.LinkSetUp(iface)
return netlink.AddrAdd(iface, ipaddr)
}
func DelIp(ip, iface string) error {
link, err := netlink.LinkByName(iface)
if err != nil {
return err
}
addr, err := netlink.ParseAddr(ip)
if err != nil {
return err
}
return netlink.AddrDel(link, addr)
}
func AssignLinkLocalIP(link netlink.Link) error {
ifaceName := link.Attrs().Name
iface, err := net.InterfaceByName(ifaceName)
if err != nil {
log.Error("could not get information about interface")
return err
}
addrs, err := iface.Addrs()
if err != nil {
log.Error("Error fetching existing ip on interface")
}
for _, addr := range addrs {
if addr.String()[:7] == "169.254" {
log.Info("Link Local IP already set on interface")
return nil
}
}
randSource, err := getPseudoRandomGenerator(link.Attrs().HardwareAddr)
if err != nil {
return err
}
// try a random address upto 10 times
for i := 0; i < 10; i++ {
randGenerator := rand.New(*randSource)
randomNum := randGenerator.Uint32()
dstIP := getNewIPV4LLAddr(randomNum)
if dstIP[2] == 0 || dstIP[2] == 255 {
i--
continue
}
_, _, err := arping.PingOverIfaceByName(dstIP, ifaceName)
if err != nil {
// this ip is not being used
addr, err := netlink.ParseAddr(dstIP.String() + "/16")
if err != nil {
log.Errorf("error while parsing ipv4ll addr, err = %v", err)
return err
}
if err := netlink.AddrAdd(link, addr); err != nil {
log.Error("ipv4ll addr add failed")
return err
}
log.Infof("Set %s on %s", dstIP.String(), link.Attrs().Name)
return nil
}
}
log.Error("Could not find a suitable ipv4ll")
return fmt.Errorf("Could not find a suitable ipv4ll")
}
func BindAndSetup(veth netlink.Link, ips string) error {
addr, err := netlink.ParseAddr(ips)
if err != nil {
return err
}
if err := netlink.AddrAdd(veth, addr); err != nil {
return err
}
if err := netlink.LinkSetUp(veth); err != nil {
return err
}
return nil
}
func setupIface(link netlink.Link, cfg Cfg) error {
// up loopback
lo, err := netlink.LinkByName("lo")
if err != nil {
return fmt.Errorf("lo interface: %v", err)
}
if err := netlink.LinkSetUp(lo); err != nil {
return fmt.Errorf("up veth: %v", err)
}
addr, err := netlink.ParseAddr(cfg.IP)
if err != nil {
return fmt.Errorf("parse IP: %v", err)
}
return netlink.AddrAdd(link, addr)
}
func applyAddress(address string, link netlink.Link, netConf InterfaceConfig) error {
addr, err := netlink.ParseAddr(address)
if err != nil {
return err
}
if err := netlink.AddrAdd(link, addr); err == syscall.EEXIST {
//Ignore this error
} else if err != nil {
log.Errorf("addr add failed: %v", err)
} else {
log.Infof("Set %s on %s", netConf.Address, link.Attrs().Name)
}
return nil
}
// initBridge creates a bridge if it does not exist
func initBridge() error {
_, err := net.InterfaceByName(bridgeName)
if err == nil {
return nil
}
if !strings.Contains(err.Error(), "no such network interface") {
return err
}
// Create *netlink.Bridge object
la := netlink.NewLinkAttrs()
la.Name = bridgeName
la.MTU = mtu
br := &netlink.Bridge{LinkAttrs: la}
if err := netlink.LinkAdd(br); err != nil {
return fmt.Errorf("Bridge creation failed: %v", err)
}
// Setup ip address for bridge
addr, err := netlink.ParseAddr(ipAddr)
if err != nil {
return fmt.Errorf("Parsing address %s: %v", ipAddr, err)
}
if err := netlink.AddrAdd(br, addr); err != nil {
return fmt.Errorf("Adding address %v to bridge %s failed: %v", addr, bridgeName, err)
}
// Validate that the IPAddress is there!
_, err = getIfaceAddr(bridgeName)
if err != nil {
return fmt.Errorf("No IP address found on bridge %s", bridgeName)
}
// Add NAT rules for iptables
if err = natOut(ipAddr, iptables.Insert); err != nil {
return fmt.Errorf("Could not set NAT rules for bridge %s", bridgeName)
}
// Bring the bridge up
if err := netlink.LinkSetUp(br); err != nil {
return err
}
return nil
}
func bridgeCreate() {
la := netlink.NewLinkAttrs()
la.Name = "mycbridge"
mybridge := &netlink.Bridge{la}
err := netlink.LinkAdd(mybridge)
if err != nil {
panic(err)
}
addr, _ := netlink.ParseAddr("172.19.80.1/24")
err = netlink.AddrAdd(mybridge, addr)
if err != nil {
panic(err)
}
err = netlink.LinkSetUp(mybridge)
if err != nil {
panic(err)
}
}
func setUpVLan(cid, ips string, pid int, veth netlink.Link) bool {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
origns, err := netns.Get()
if err != nil {
logs.Info("Get orignal namespace failed", err)
return false
}
defer origns.Close()
ns, err := netns.GetFromPid(pid)
if err != nil {
logs.Info("Get container namespace failed", err)
return false
}
netns.Set(ns)
defer ns.Close()
defer netns.Set(origns)
addr, err := netlink.ParseAddr(ips)
if err != nil {
logs.Info("Parse CIDR failed", err)
return false
}
if err := netlink.AddrAdd(veth, addr); err != nil {
logs.Info("Add addr to veth failed", err)
return false
}
if err := netlink.LinkSetUp(veth); err != nil {
logs.Info("Setup veth failed", err)
return false
}
logs.Info("Add vlan device success", cid[:12])
return true
}
/* Configure @iface with address @cidr. Deletes any existing addresses */
func SetAddr(iface string, cidr string) {
if strings.Index(cidr, "/") == -1 {
/* No scope given: assume full netmask */
cidr = cidr + "/32"
}
addr, err := netlink.ParseAddr(cidr)
if err != nil {
log.Fatalf("failed to parse %s interface address %s: %s", iface, cidr, err)
}
link, err := netlink.LinkByName(iface)
if err != nil {
log.Fatalf("failed to look up interface %s: %s", iface, err)
}
addrs, err := netlink.AddrList(link, 0)
if err != nil {
log.Fatalf("failed to get %s interface addresses: %s", iface, err)
}
already_configured := false
for _, old_addr := range addrs {
/* Test if already configured, otherwise EEXISTS */
if old_addr.Equal(*addr) {
already_configured = true
} else {
log.Printf("removing %s from %s", old_addr.IPNet, iface)
if err := netlink.AddrDel(link, &old_addr); err != nil {
log.Fatalf("failed to remove address from %s: %s", iface, err)
}
}
}
log.Printf("configuring %s on %s", addr.IPNet, iface)
if !already_configured {
if err := netlink.AddrAdd(link, addr); err != nil {
log.Fatalf("failed to add address to %s: %s", iface, err)
}
}
}
//netlink is not avaible in MAC OS, build fail.
func AddContainerNetworking() {
if CliIF == "" {
log.Fatal("the host-interface is missing,please give one")
}
if ok := utils.ValidateHostIface(CliIF); !ok {
log.Fatalf("the host-interface [ %s ] was not found.", CliIF)
}
hostmacvlanname, _ := utils.GenerateRandomName(hostprefix, hostlen)
hostEth, _ := netlink.LinkByName(CliIF)
//create the macvlan device
macvlandev := &netlink.Macvlan{
LinkAttrs: netlink.LinkAttrs{
Name: hostmacvlanname,
ParentIndex: hostEth.Attrs().Index,
},
Mode: netlink.MACVLAN_MODE_BRIDGE,
}
if err := netlink.LinkAdd(macvlandev); err != nil {
log.Fatalf("failed to create Macvlan: [ %v ] with the error: %s", macvlandev.Attrs().Name, err)
}
// log.Infof("Created Macvlan port: [ %s ] using the mode: [ %s ]", macvlan.Name, macvlanMode)
// ugly, actually ,can get the ns from netns.getfromDocker. the netns have many function, netns.getformpid
// netns.getfromdocker the arg can not be the container name
dockerPid := utils.DockerPid(CliCName)
//the macvlandev can be use directly, don't get netlink.byname again.
netlink.LinkSetNsPid(macvlandev, dockerPid)
runtime.LockOSThread()
defer runtime.UnlockOSThread()
//get root network namespace
origns, _ := netns.Get()
defer origns.Close()
//enter the docker container network
dockerNS, _ := netns.GetFromPid(dockerPid)
defer dockerNS.Close()
netns.Set(dockerNS)
// use macvlandev can cause error,need type assertion. netlink.Macvlan not must be netlink.Link,fmz(realy? the vlan action add)
// it is wrong,
macvlandev1, _ := netlink.LinkByName(macvlandev.Attrs().Name)
// when the eth is up, set name fail,: Device or resource busy
netlink.LinkSetDown(macvlandev1)
netlink.LinkSetName(macvlandev1, "eth1")
addr, err := netlink.ParseAddr(CliIP)
if err != nil {
log.Fatalf("failed to parse the ip address %v", CliIP)
}
netlink.AddrAdd(macvlandev1, addr)
netlink.LinkSetUp(macvlandev1)
/* set the default route, have some problem. Dst == 0.0.0.0/0? no
defaultgw := &netlink.Route{
Dst: nil,
}
netlink.RouteDel(defaultgw)
ip, _ := net.ParseIP("8.8.8.8")
routes, _ := netlink.RouteGet(ip)
for _, r := range routes {
netlink.RouteDel(&r)
}
*/
//if use ip instruction, it also can config the container, --privileged have no effect.
// The sublime test code(test this function) is strange, it only can avaiable in first time. And then fail(even need to reboot)
// got it,
//following code successfully delete the default route in docker container,but error in my host ,no such process
routes, _ := netlink.RouteList(nil, netlink.FAMILY_V4)
for _, r := range routes {
if r.Dst == nil {
if err := netlink.RouteDel(&r); err != nil {
log.Warnf("delete the default error: ", err)
}
}
}
if CligwIP == "" {
log.Fatal("container gw is null")
}
defaultRoute := &netlink.Route{
Dst: nil,
Gw: net.ParseIP(CligwIP),
LinkIndex: macvlandev1.Attrs().Index,
}
if err := netlink.RouteAdd(defaultRoute); err != nil {
log.Warnf("create default route error: ", err)
}
netns.Set(origns)
}
func main() {
if len(os.Args) != 3 {
panic("not enough args")
}
intf, err := net.InterfaceByName(os.Args[1])
if err != nil {
panic(err)
}
containerID := os.Args[2]
log.Infof("Requesting for container ID %q on interface %q", containerID, intf.Name)
contPid := getContainerInfo(containerID)
log.Infof("Container has PID %s", contPid)
runtime.LockOSThread()
f, err := os.Open(fmt.Sprintf("/proc/%s/ns/net", contPid))
if err != nil {
panic(err)
}
if _, _, err := unix.RawSyscall(unix.SYS_SETNS, f.Fd(), syscall.CLONE_NEWNET, 0); err != 0 {
panic(err)
}
link, err := netlink.LinkByName("eth0")
if err != nil {
panic(err)
}
chaddr := link.Attrs().HardwareAddr
p := dhcp.RequestPacket(dhcp.Discover, chaddr, nil, []byte{1, 2, 3, 4}, true, []dhcp.Option{})
conn, err := net.ListenPacket("udp", ":68")
if err != nil {
panic(err)
}
ipc := ipv4.NewPacketConn(conn)
ipc.SetControlMessage(ipv4.FlagInterface, true)
to := &net.UDPAddr{
IP: net.IPv4(255, 255, 255, 255),
Port: 67,
}
eth0, err := net.InterfaceByName("eth0")
if err != nil {
panic(err)
}
if _, err := ipc.WriteTo(p, &ipv4.ControlMessage{IfIndex: eth0.Index}, to); err != nil {
panic(err)
}
newPacket := dhcp.NewPacket(dhcp.BootReply)
for {
_, cm, _, err := ipc.ReadFrom(newPacket)
if err != nil {
panic(err)
}
if cm != nil && cm.IfIndex == eth0.Index {
log.Infof("Received reply: %q", newPacket.YIAddr().String())
break
}
}
addr, err := netlink.ParseAddr(newPacket.YIAddr().String() + "/16")
if err != nil {
panic(err)
}
log.Infof("Setting interface eth0...")
if err := netlink.AddrAdd(link, addr); err != nil {
panic(err)
}
}
/*
Bgp serve routine does the following:
1) Creates inb01 router port
2) Add MyBgp endpoint
3) Kicks off routines to monitor route updates and peer state
*/
func (self *OfnetBgp) StartProtoServer(routerInfo *OfnetProtoRouterInfo) error {
log.Infof("Starting the Bgp Server with %v", routerInfo)
//go routine to start gobgp server
var len uint
var err error
self.routerIP, len, err = ParseCIDR(routerInfo.RouterIP)
as, _ := strconv.Atoi(routerInfo.As)
self.myBgpAs = uint32(as)
self.modRibCh = make(chan *api.Path, 16)
self.advPathCh = make(chan *api.Path, 16)
timeout := grpc.WithTimeout(time.Second)
conn, err := grpc.Dial("127.0.0.1:8080", timeout, grpc.WithBlock(), grpc.WithInsecure())
if err != nil {
log.Fatal(err)
}
self.cc = conn
defer self.cc.Close()
client := api.NewGobgpApiClient(self.cc)
if client == nil {
log.Errorf("Invalid Gobgpapi client")
return errors.New("Error creating Gobgpapiclient")
}
path := &api.Path{
Pattrs: make([][]byte, 0),
}
path.Nlri, _ = bgp.NewIPAddrPrefix(uint8(32), self.routerIP).Serialize()
n, _ := bgp.NewPathAttributeNextHop("0.0.0.0").Serialize()
path.Pattrs = append(path.Pattrs, n)
origin, _ := bgp.NewPathAttributeOrigin(bgp.BGP_ORIGIN_ATTR_TYPE_INCOMPLETE).Serialize()
path.Pattrs = append(path.Pattrs, origin)
log.Debugf("Creating the loopback port ")
err = self.agent.ovsDriver.CreatePort(self.intfName, "internal", 1)
if err != nil {
log.Errorf("Error creating the port", err)
}
intfIP := fmt.Sprintf("%s/%d", self.routerIP, len)
log.Debugf("Creating inb01 with ", intfIP)
ofPortno, _ := self.agent.ovsDriver.GetOfpPortNo(self.intfName)
link, err := netlink.LinkByName(self.intfName)
if err != nil {
log.Errorf("error finding link by name %v", self.intfName)
return err
}
linkIP, err := netlink.ParseAddr(intfIP)
if err != nil {
log.Errorf("invalid ip ", intfIP)
}
netlink.AddrAdd(link, linkIP)
if link == nil || ofPortno == 0 {
log.Errorf("Error fetching %v information", self.intfName, link, ofPortno)
return errors.New("Unable to fetch inb01 info")
}
intf, _ := net.InterfaceByName(self.intfName)
epreg := &OfnetEndpoint{
EndpointID: self.routerIP,
EndpointType: "internal-bgp",
IpAddr: net.ParseIP(self.routerIP),
IpMask: net.ParseIP("255.255.255.255"),
VrfId: 0, // FIXME set VRF correctly
MacAddrStr: intf.HardwareAddr.String(), //link.Attrs().HardwareAddr.String(),
Vlan: 1,
PortNo: ofPortno,
Timestamp: time.Now(),
}
// Add the endpoint to local routing table
self.agent.endpointDb[self.routerIP] = epreg
self.agent.localEndpointDb[epreg.PortNo] = epreg
fmt.Println(epreg)
err = self.agent.datapath.AddLocalEndpoint(*epreg)
//Add bgp router id as well
bgpGlobalCfg := &bgpconf.Global{}
setDefaultGlobalConfigValues(bgpGlobalCfg)
bgpGlobalCfg.GlobalConfig.RouterId = net.ParseIP(self.routerIP)
bgpGlobalCfg.GlobalConfig.As = self.myBgpAs
self.bgpServer.SetGlobalType(*bgpGlobalCfg)
self.advPathCh <- path
//monitor route updates from peer
go self.monitorBest()
//monitor peer state
go self.monitorPeer()
self.start <- true
for {
select {
case p := <-self.modRibCh:
err = self.modRib(p)
if err != nil {
log.Error("failed to mod rib: ", err)
}
case <-self.stop:
return nil
}
}
return nil
}
/*
Bgp serve routine does the following:
1) Creates inb01 router port
2) Add MyBgp endpoint
3) Kicks off routines to monitor route updates and peer state
*/
func (self *OfnetBgp) StartProtoServer(routerInfo *OfnetProtoRouterInfo) error {
log.Infof("Starting the Bgp Server with %v", routerInfo)
//go routine to start gobgp server
var len uint
var err error
self.routerIP, len, err = ParseCIDR(routerInfo.RouterIP)
as, _ := strconv.Atoi(routerInfo.As)
self.myBgpAs = uint32(as)
timeout := grpc.WithTimeout(time.Second)
conn, err := grpc.Dial("127.0.0.1:50051", timeout, grpc.WithBlock(), grpc.WithInsecure())
if err != nil {
log.Fatal(err)
}
self.cc = conn
log.Debugf("Creating the loopback port ")
err = self.agent.ovsDriver.CreatePort(self.intfName, "internal", 1)
if err != nil {
log.Errorf("Error creating the port: %v", err)
}
intfIP := fmt.Sprintf("%s/%d", self.routerIP, len)
log.Debugf("Creating inb01 with ", intfIP)
ofPortno, _ := self.agent.ovsDriver.GetOfpPortNo(self.intfName)
link, err := netlink.LinkByName(self.intfName)
if err != nil {
log.Errorf("error finding link by name %v", self.intfName)
return err
}
linkIP, err := netlink.ParseAddr(intfIP)
if err != nil {
log.Errorf("invalid ip: %s", intfIP)
return err
}
netlink.AddrAdd(link, linkIP)
netlink.LinkSetUp(link)
if link == nil || ofPortno == 0 {
log.Errorf("Error fetching %v/%v/%v information", self.intfName, link, ofPortno)
return errors.New("Unable to fetch inb01 info")
}
intf, _ := net.InterfaceByName(self.intfName)
vrf := "default"
epid := self.agent.getEndpointIdByIpVrf(net.ParseIP(self.routerIP), vrf)
default_vlan := uint16(1)
_, ok := self.agent.createVrf(vrf)
if !ok {
log.Errorf("Error Creating default vrf for Bgp")
return errors.New("Error creating default vrf")
}
self.agent.vlanVrf[default_vlan] = &vrf
ep := &OfnetEndpoint{
EndpointID: epid,
EndpointType: "internal-bgp",
IpAddr: net.ParseIP(self.routerIP),
IpMask: net.ParseIP("255.255.255.255"),
Vrf: "default", // FIXME set VRF correctly
MacAddrStr: intf.HardwareAddr.String(), //link.Attrs().HardwareAddr.String(),
Vlan: default_vlan,
PortNo: ofPortno,
Timestamp: time.Now(),
}
// Add the endpoint to local routing table
err = self.agent.datapath.AddLocalEndpoint(*ep)
if err != nil {
log.Errorf("Error Adding Local Bgp Endpoint for endpoint %+v,err: %v", ep, err)
return err
}
self.agent.endpointDb.Set(epid, ep)
self.agent.localEndpointDb.Set(string(ep.PortNo), ep)
// global configuration
global := &bgpconf.Global{
Config: bgpconf.GlobalConfig{
As: self.myBgpAs,
RouterId: self.routerIP,
Port: 179,
},
}
if err := self.bgpServer.Start(global); err != nil {
return err
}
//monitor route updates from peer, peer state
go self.watch()
// register for link ups on uplink and inb01 intf
self.start <- true
return nil
}
func AddVlannetwork(etcdval string, vlanid string, containerName string) {
ss := strings.Split(etcdval, ",")
hostif := ss[0]
if ok := utils.ValidateHostIface(hostif); !ok {
log.Warnf("the host interface not exist")
return
}
vlandevName := hostif + "." + vlanid
hostEth, _ := netlink.LinkByName(hostif)
intvlanid, err := strconv.Atoi(vlanid)
if err != nil {
log.Warnf("the vlan id convert error: \n")
return
}
var vlandev *netlink.Vlan
if ok := utils.ValidateHostIface(vlandevName); ok {
} else {
//not exist ,create the vlan device
vlandev = &netlink.Vlan{
LinkAttrs: netlink.LinkAttrs{
Name: vlandevName,
ParentIndex: hostEth.Attrs().Index,
},
VlanId: intvlanid,
}
if err := netlink.LinkAdd(vlandev); err != nil {
log.Warnf("failed to create vlandev: [ %v ] with the error: %s", vlandev, err)
return
}
}
netlink.LinkSetUp(vlandev)
macvlanname, _ := utils.GenerateRandomName("vlan"+vlanid, 5)
//create the macvlan device
macvlandev := &netlink.Macvlan{
LinkAttrs: netlink.LinkAttrs{
Name: macvlanname,
ParentIndex: vlandev.Attrs().Index,
},
Mode: netlink.MACVLAN_MODE_BRIDGE,
}
if err := netlink.LinkAdd(macvlandev); err != nil {
log.Warnf("failed to create Macvlan: [ %v ] with the error: %s", macvlandev, err)
return
}
dockerPid := utils.DockerPid(containerName)
//the macvlandev can be use directly, don't get netlink.byname again.
netlink.LinkSetNsPid(macvlandev, dockerPid)
runtime.LockOSThread()
defer runtime.UnlockOSThread()
//get root network naAddVlannetworkmespace
origns, _ := netns.Get()
defer origns.Close()
//enter the docker container network
dockerNS, _ := netns.GetFromPid(dockerPid)
defer dockerNS.Close()
netns.Set(dockerNS)
netlink.LinkSetDown(macvlandev)
netlink.LinkSetName(macvlandev, "eth1")
_, network, _ := net.ParseCIDR(ss[1])
if _, ok := ipallocs[vlanid]; !ok {
log.Fatalf("the ipallocator is null \n")
}
ip, _ := ipallocs[vlanid].RequestIP(network, nil)
ind := strings.LastIndex(ss[1], "/")
ipstring := ip.String() + ss[1][ind:]
addr, err := netlink.ParseAddr(ipstring)
netlink.AddrAdd(macvlandev, addr)
netlink.LinkSetUp(macvlandev)
/*
routes, _ := netlink.RouteList(nil, netlink.FAMILY_V4)
for _, r := range routes {
if r.Dst == nil {
if err := netlink.RouteDel(&r); err != nil {
log.Warnf("delete the default error: ", err)
}
}
}
if CligwIP == "" {
log.Fatal("container gw is null")
}
defaultRoute := &netlink.Route{
Dst: nil,
Gw: net.ParseIP(CligwIP),
LinkIndex: macvlandev1.Attrs().Index,
}
if err := netlink.RouteAdd(defaultRoute); err != nil {
log.Warnf("create default route error: ", err)
}
*/
netns.Set(origns)
}
func (v *veth) initialize(config *network) error {
peer := config.TempVethPeerName
if peer == "" {
return fmt.Errorf("peer is not specified")
}
child, err := netlink.LinkByName(peer)
if err != nil {
return err
}
if err := netlink.LinkSetDown(child); err != nil {
return err
}
if err := netlink.LinkSetName(child, config.Name); err != nil {
return err
}
// get the interface again after we changed the name as the index also changes.
if child, err = netlink.LinkByName(config.Name); err != nil {
return err
}
if config.MacAddress != "" {
mac, err := net.ParseMAC(config.MacAddress)
if err != nil {
return err
}
if err := netlink.LinkSetHardwareAddr(child, mac); err != nil {
return err
}
}
ip, err := netlink.ParseAddr(config.Address)
if err != nil {
return err
}
if err := netlink.AddrAdd(child, ip); err != nil {
return err
}
if config.IPv6Address != "" {
ip6, err := netlink.ParseAddr(config.IPv6Address)
if err != nil {
return err
}
if err := netlink.AddrAdd(child, ip6); err != nil {
return err
}
}
if err := netlink.LinkSetMTU(child, config.Mtu); err != nil {
return err
}
if err := netlink.LinkSetUp(child); err != nil {
return err
}
if config.Gateway != "" {
gw := net.ParseIP(config.Gateway)
if err := netlink.RouteAdd(&netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
LinkIndex: child.Attrs().Index,
Gw: gw,
}); err != nil {
return err
}
}
if config.IPv6Gateway != "" {
gw := net.ParseIP(config.IPv6Gateway)
if err := netlink.RouteAdd(&netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
LinkIndex: child.Attrs().Index,
Gw: gw,
}); err != nil {
return err
}
}
return nil
}
// configureInterface is used to configure an individual interface against a
// matched configuration. It sets up the addresses, the MTU, and invokes DHCP if
// necessary.
func configureInterface(link netlink.Link, netconf *kurmaNetworkInterface) error {
linkName := link.Attrs().Name
addressConfigured := true
// configure using DHCP
if netconf.DHCP {
cmd := exec.Command("udhcpc", "-i", linkName, "-t", "20", "-n")
cmd.Stdin = nil
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to configure %s with DHCP: %v", linkName, err)
}
addressConfigured = true
}
// single address
if netconf.Address != "" {
addr, err := netlink.ParseAddr(netconf.Address)
if err != nil {
return fmt.Errorf("failed to parse address %q on %s", netconf.Address, linkName)
}
if err := netlink.AddrAdd(link, addr); err != nil {
return fmt.Errorf("failed to configure address %q on %s: %v",
netconf.Address, linkName, err)
}
addressConfigured = true
}
// list of addresses
for _, address := range netconf.Addresses {
addr, err := netlink.ParseAddr(address)
if err != nil {
return fmt.Errorf("failed to parse address %q on %s", address, linkName)
}
if err := netlink.AddrAdd(link, addr); err != nil {
return fmt.Errorf("failed to configure address %q on %s: %v",
address, linkName, err)
}
addressConfigured = true
}
if !addressConfigured {
return fmt.Errorf("no address configured to %s: unable to set link up", linkName)
}
if netconf.MTU > 0 {
if err := netlink.LinkSetMTU(link, netconf.MTU); err != nil {
return fmt.Errorf("failed to set mtu on %s: %v", linkName, err)
}
}
// verify it is up at the end
if link.Attrs().Flags&net.FlagUp == 0 {
if err := netlink.LinkSetUp(link); err != nil {
return fmt.Errorf("failed to set link %s up: %v", linkName, err)
}
}
return nil
}
/*
Bgp serve routine does the following:
1) Creates inb01 router port
2) Add MyBgp endpoint
3) Kicks off routines to monitor route updates and peer state
*/
func (self *OfnetBgp) StartProtoServer(routerInfo *OfnetProtoRouterInfo) error {
log.Infof("Starting the Bgp Server with %v", routerInfo)
//go routine to start gobgp server
var len uint
var err error
self.routerIP, len, err = ParseCIDR(routerInfo.RouterIP)
as, _ := strconv.Atoi(routerInfo.As)
self.myBgpAs = uint32(as)
log.Infof("Starting the Bgp Server for AS %v", self.myBgpAs)
self.modRibCh = make(chan *api.Path, 16)
self.advPathCh = make(chan *api.Path, 16)
timeout := grpc.WithTimeout(time.Second)
conn, err := grpc.Dial("127.0.0.1:50051", timeout, grpc.WithBlock(), grpc.WithInsecure())
if err != nil {
log.Infof("GRPC timeout")
log.Fatal(err)
}
self.cc = conn
defer self.cc.Close()
client := api.NewGobgpApiClient(self.cc)
if client == nil {
log.Errorf("Invalid Gobgpapi client")
return errors.New("Error creating Gobgpapiclient")
}
path := &api.Path{
Pattrs: make([][]byte, 0),
}
path.Nlri, _ = bgp.NewIPAddrPrefix(uint8(32), self.routerIP).Serialize()
n, _ := bgp.NewPathAttributeNextHop("0.0.0.0").Serialize()
path.Pattrs = append(path.Pattrs, n)
origin, _ := bgp.NewPathAttributeOrigin(bgp.BGP_ORIGIN_ATTR_TYPE_INCOMPLETE).Serialize()
path.Pattrs = append(path.Pattrs, origin)
log.Infof("Creating the loopback port ")
err = self.agent.ovsDriver.CreatePort(self.intfName, "internal", 1)
if err != nil {
log.Errorf("Error creating the port: %v", err)
}
defer self.agent.ovsDriver.DeletePort(self.intfName)
intfIP := fmt.Sprintf("%s/%d", self.routerIP, len)
log.Infof("Creating inb01 with ", intfIP)
ofPortno, _ := self.agent.ovsDriver.GetOfpPortNo(self.intfName)
link, err := netlink.LinkByName(self.intfName)
if err != nil {
log.Errorf("error finding link by name %v", self.intfName)
return err
}
linkIP, err := netlink.ParseAddr(intfIP)
if err != nil {
log.Errorf("invalid ip: %s", intfIP)
return err
}
netlink.AddrAdd(link, linkIP)
netlink.LinkSetUp(link)
if link == nil || ofPortno == 0 {
log.Errorf("Error fetching %v/%v/%v information", self.intfName, link, ofPortno)
return errors.New("Unable to fetch inb01 info")
}
intf, _ := net.InterfaceByName(self.intfName)
log.Infof("routerIP ", self.routerIP)
epid := self.agent.getEndpointIdByIpVrf(net.ParseIP(self.routerIP), "default")
log.Infof("Create VRF ")
_, ok := self.agent.createVrf("default")
if !ok {
log.Errorf("Error Creating default vrf for Bgp")
return errors.New("Error creating default vrf")
}
epreg := &OfnetEndpoint{
EndpointID: epid,
EndpointType: "internal-bgp",
IpAddr: net.ParseIP(self.routerIP),
IpMask: net.ParseIP("255.255.255.255"),
Vrf: "default", // FIXME set VRF correctly
MacAddrStr: intf.HardwareAddr.String(), //link.Attrs().HardwareAddr.String(),
Vlan: 1,
PortNo: ofPortno,
Timestamp: time.Now(),
}
// Add the endpoint to local routing table
self.agent.endpointDb[epid] = epreg
//log.Infof("self.agent.endpointDb[epid] %v " , self.agent.endpointDb[epid] )
self.agent.localEndpointDb[epreg.PortNo] = epreg
fmt.Println(epreg)
err = self.agent.datapath.AddLocalEndpoint(*epreg)
//Add bgp router id as well
bgpGlobalCfg := &bgpconf.Global{}
setDefaultGlobalConfigValues(bgpGlobalCfg)
bgpGlobalCfg.Config.RouterId = self.routerIP
bgpGlobalCfg.Config.As = self.myBgpAs
//self.bgpServer.SetGlobalType(*bgpGlobalCfg)
_, err2 := client.StartServer(context.Background(), &api.StartServerRequest{
Global: &api.Global{
As: uint32(self.myBgpAs),
RouterId: self.routerIP,
ListenPort: int32(epreg.PortNo),
ListenAddresses: []string{self.routerIP},
MplsLabelMin: uint32(16000),
MplsLabelMax: uint32(1048575),
},
})
log.Info("BGP -StartServer ERR %v", err2)
self.advPathCh <- path
log.Infof("start monitoring ")
//monitor route updates from peer
go self.monitorBest()
//monitor peer state
go self.monitorPeer()
self.start <- true
for {
select {
case p := <-self.modRibCh:
err = self.modRib(p)
if err != nil {
log.Error("failed to mod rib: ", err)
}
case <-self.stop:
return nil
}
}
}