func setIfNs(ifname string, pid int) error {
link, err := netlink.LinkByName(ifname)
if err != nil {
if !strings.Contains(err.Error(), "Link not found") {
log.Errorf("unable to find link %q. Error: %q", ifname, err)
return err
}
// try once more as sometimes (somehow) link creation is taking
// sometime, causing link not found error
time.Sleep(1 * time.Second)
link, err = netlink.LinkByName(ifname)
if err != nil {
log.Errorf("unable to find link %q. Error %q", ifname, err)
return err
}
}
err = netlink.LinkSetNsPid(link, pid)
if err != nil {
log.Errorf("unable to move interface '%s' to pid %d. Error: %s",
ifname, pid, err)
}
return err
}
func (v *veth) create(n *network, nspid int) (err error) {
tmpName, err := v.generateTempPeerName()
if err != nil {
return err
}
n.TempVethPeerName = tmpName
if n.Bridge == "" {
return fmt.Errorf("bridge is not specified")
}
veth := &netlink.Veth{
LinkAttrs: netlink.LinkAttrs{
Name: n.HostInterfaceName,
TxQLen: n.TxQueueLen,
},
PeerName: n.TempVethPeerName,
}
if err := netlink.LinkAdd(veth); err != nil {
return err
}
defer func() {
if err != nil {
netlink.LinkDel(veth)
}
}()
if err := v.attach(&n.Network); err != nil {
return err
}
child, err := netlink.LinkByName(n.TempVethPeerName)
if err != nil {
return err
}
return netlink.LinkSetNsPid(child, nspid)
}
func moveToNS(pid int, ifname string) error {
// find the link
link, err := getLink(ifname)
if err != nil {
log.Errorf("unable to find link %q. Error %q", ifname, err)
return err
}
// move to the desired netns
err = netlink.LinkSetNsPid(link, pid)
if err != nil {
log.Errorf("unable to move interface %s to pid %d. Error: %s",
ifname, pid, err)
return err
}
return nil
}
/* setup()
*/
func setup(pid int, brName string) error {
br, e := netlink.LinkByName(brName)
if e != nil {
return fmt.Errorf("Host bridge not found: %v", e)
}
// Network interface names.
hsIfName := fmt.Sprintf("veth%d", pid) // Host side interface
nsIfName := "veth0"
// Assign host side interface to bridge.
linkAttrs := netlink.NewLinkAttrs()
linkAttrs.Name = hsIfName
linkAttrs.MasterIndex = br.Attrs().Index
// Create interface pair.
hsIf := &netlink.Veth{LinkAttrs: linkAttrs, PeerName: nsIfName}
if e := netlink.LinkAdd(hsIf); e != nil {
return fmt.Errorf("Failed to create veth pair: %v", e)
}
// Get namespace side interface handle.
nsIf, e := netlink.LinkByName(nsIfName)
if e != nil {
netlink.LinkDel(hsIf)
return fmt.Errorf("Failed to get namespace iface: %v", e)
}
// Attach network interface to namespace.
if e := netlink.LinkSetNsPid(nsIf, pid); e != nil {
netlink.LinkDel(hsIf)
return fmt.Errorf("Failed to attach namespace iface: %v", e)
}
// Bring up host side interface.
if e := netlink.LinkSetUp(hsIf); e != nil {
netlink.LinkDel(hsIf)
return fmt.Errorf("Failed to bring up host iface: %v", e)
}
return nil
}
func AddVlan(vethName, ips, cid string) bool {
lock.Lock()
defer lock.Unlock()
container, err := g.Docker.InspectContainer(cid)
if err != nil {
logs.Info("VLanSetter inspect docker failed", err)
return false
}
veth, err := AddMacVlanDevice(vethName, cid)
if err != nil {
logs.Info("Create macvlan device failed", err)
return false
}
if err := netlink.LinkSetNsPid(veth, container.State.Pid); err != nil {
logs.Info("Set macvlan device into container failed", err)
DelVlan(veth)
return false
}
return setUpVLan(cid, ips, container.State.Pid, veth)
}
func AddVLan(vethName, ips, cid string) bool {
lock.Lock()
defer lock.Unlock()
device, _ := Devices.Get(cid, 0)
logs.Info("Add new VLan to", vethName, cid[:12])
container, err := g.Docker.InspectContainer(cid)
if err != nil {
logs.Info("VLanSetter inspect docker failed", err)
return false
}
parent, err := netlink.LinkByName(device)
if err != nil {
logs.Info("Get parent NIC failed", err)
return false
}
veth := &netlink.Macvlan{
LinkAttrs: netlink.LinkAttrs{Name: vethName, ParentIndex: parent.Attrs().Index},
Mode: netlink.MACVLAN_MODE_BRIDGE,
}
if err := netlink.LinkAdd(veth); err != nil {
logs.Info("Create macvlan device failed", err)
return false
}
if err := netlink.LinkSetNsPid(veth, container.State.Pid); err != nil {
logs.Info("Set macvlan device into container failed", err)
delVLan(veth)
return false
}
return setUpVLan(cid, ips, container.State.Pid, veth)
}
func AddDHCPNetwork() {
if ok := utils.ValidateHostIface(flat.CliIF); !ok {
log.Fatalf("the host-interface [ %s ] was not found.", flat.CliIF)
}
hostmacvlanname, _ := utils.GenerateRandomName(hostprefix, hostlen)
hostEth, _ := netlink.LinkByName(flat.CliIF)
//create the macvlan device
macvlandev := &netlink.Macvlan{
LinkAttrs: netlink.LinkAttrs{
Name: hostmacvlanname,
ParentIndex: hostEth.Attrs().Index,
},
Mode: netlink.MACVLAN_MODE_BRIDGE,
}
if err := netlink.LinkAdd(macvlandev); err != nil {
log.Fatalf("failed to create Macvlan: [ %v ] with the error: %s", macvlandev.Attrs().Name, err)
}
dockerPid := utils.DockerPid(flat.CliCName)
netlink.LinkSetNsPid(macvlandev, dockerPid)
runtime.LockOSThread()
defer runtime.UnlockOSThread()
//get root network namespace
origns, _ := netns.Get()
defer origns.Close()
//enter the docker container network
dockerNS, _ := netns.GetFromPid(dockerPid)
defer dockerNS.Close()
//get the dochclient name
dhcpClientPath := "/home/fmzhen/go/src/github.com/fmzhen/docker-macvlan/macvlan/dhcp/dhcpclient.sh"
out, err := exec.Command(dhcpClientPath).Output()
if err != nil {
log.Fatal("exec the dhcpclient.sh error: ", err)
}
dhcpClient := string(out)
// like ip netns exec xxx, just the netns ,so the command in host can exec
netns.Set(dockerNS)
// use macvlandev can cause error,need type assertion. netlink.Macvlan not must be netlink.Link,fmz
macvlandev1, _ := netlink.LinkByName(macvlandev.Attrs().Name)
netlink.LinkSetDown(macvlandev1)
netlink.LinkSetName(macvlandev1, "eth1")
netlink.LinkSetUp(macvlandev1)
//delete the default route
routes, _ := netlink.RouteList(nil, netlink.FAMILY_V4)
for _, r := range routes {
if r.Dst == nil {
if err := netlink.RouteDel(&r); err != nil {
log.Warnf("delete the default error: ", err)
}
}
}
//it doesn't work, the problem at the atgs don't pass to the shell script
dhcpReqpath := "/home/fmzhen/go/src/github.com/fmzhen/docker-macvlan/macvlan/dhcp/dhcpReq.sh"
exec.Command(dhcpReqpath, dhcpClient, string(dockerPid), flat.CliCName).Run()
netns.Set(origns)
}
// setIfAttrs sets the required attributes for the container interface
func setIfAttrs(pid int, ifname, cidr, newname string) error {
nsenterPath, err := osexec.LookPath("nsenter")
if err != nil {
return err
}
ipPath, err := osexec.LookPath("ip")
if err != nil {
return err
}
// find the link
link, err := getLink(ifname)
if err != nil {
log.Errorf("unable to find link %q. Error %q", ifname, err)
return err
}
// move to the desired netns
err = netlink.LinkSetNsPid(link, pid)
if err != nil {
log.Errorf("unable to move interface %s to pid %d. Error: %s",
ifname, pid, err)
return err
}
// rename to the desired ifname
nsPid := fmt.Sprintf("%d", pid)
rename, err := osexec.Command(nsenterPath, "-t", nsPid, "-n", "-F", "--", ipPath, "link",
"set", "dev", ifname, "name", newname).CombinedOutput()
if err != nil {
log.Errorf("unable to rename interface %s to %s. Error: %s",
ifname, newname, err)
return nil
}
log.Infof("Output from rename: %v", rename)
// set the ip address
assignIP, err := osexec.Command(nsenterPath, "-t", nsPid, "-n", "-F", "--", ipPath,
"address", "add", cidr, "dev", newname).CombinedOutput()
if err != nil {
log.Errorf("unable to assign ip %s to %s. Error: %s",
cidr, newname, err)
return nil
}
log.Infof("Output from ip assign: %v", assignIP)
// Finally, mark the link up
bringUp, err := osexec.Command(nsenterPath, "-t", nsPid, "-n", "-F", "--", ipPath,
"link", "set", "dev", newname, "up").CombinedOutput()
if err != nil {
log.Errorf("unable to assign ip %s to %s. Error: %s",
cidr, newname, err)
return nil
}
log.Debugf("Output from ip assign: %v", bringUp)
return nil
}
func createNetwork() error {
// Get hook data
h, err := readHookData()
if err != nil {
return err
}
// Initialize the bridge
if err := initBridge(); err != nil {
return err
}
// Create and attach local name to the bridge
localVethPair, err := vethPair(h.Pid, bridgeName)
if err != nil {
return fmt.Errorf("Getting vethpair failed for pid %d: %v", h.Pid, err)
}
if err := netlink.LinkAdd(localVethPair); err != nil {
return fmt.Errorf("Create veth pair named [ %#v ] failed: %v", localVethPair, err)
}
// Get the peer link
peer, err := netlink.LinkByName(localVethPair.PeerName)
if err != nil {
return fmt.Errorf("Getting peer interface (%s) failed: %v", localVethPair.PeerName, err)
}
// Put peer interface into the network namespace of specified PID
if err := netlink.LinkSetNsPid(peer, h.Pid); err != nil {
return fmt.Errorf("Adding peer interface to network namespace of pid %d failed: %v", h.Pid, err)
}
// Bring the veth pair up
if err := netlink.LinkSetUp(localVethPair); err != nil {
return fmt.Errorf("Bringing local veth pair [ %#v ] up failed: %v", localVethPair, err)
}
// check the bridge IPNet as it may be different than the default
brNet, err := getIfaceAddr(bridgeName)
if err != nil {
return fmt.Errorf("Retrieving IP/network of bridge %s failed: %v", bridgeName, err)
}
// Allocate an ip address for the interface
ip, ipNet, err := net.ParseCIDR(brNet.String())
if err != nil {
return fmt.Errorf("Parsing CIDR for %s failed: %v", ipAddr, err)
}
ipAllocator, err := ipallocator.New(bridgeName, stateDir, ipNet)
if err != nil {
return err
}
nsip, err := ipAllocator.Allocate(h.Pid)
if err != nil {
return fmt.Errorf("Allocating ip address failed: %v", err)
}
newIP := &net.IPNet{
IP: nsip,
Mask: ipNet.Mask,
}
// Configure the interface in the network namespace
if err := configureInterface(localVethPair.PeerName, h.Pid, newIP, ip.String()); err != nil {
return err
}
logrus.Infof("Attached veth (%s) to bridge (%s)", localVethPair.Name, bridgeName)
// save the ip to a file so other hooks can use it
if err := ioutil.WriteFile(ipfile, []byte(nsip.String()), 0755); err != nil {
return fmt.Errorf("Saving allocated ip address for container to %s failed: %v", ipfile, err)
}
return nil
}
//netlink is not avaible in MAC OS, build fail.
func AddContainerNetworking() {
if CliIF == "" {
log.Fatal("the host-interface is missing,please give one")
}
if ok := utils.ValidateHostIface(CliIF); !ok {
log.Fatalf("the host-interface [ %s ] was not found.", CliIF)
}
hostmacvlanname, _ := utils.GenerateRandomName(hostprefix, hostlen)
hostEth, _ := netlink.LinkByName(CliIF)
//create the macvlan device
macvlandev := &netlink.Macvlan{
LinkAttrs: netlink.LinkAttrs{
Name: hostmacvlanname,
ParentIndex: hostEth.Attrs().Index,
},
Mode: netlink.MACVLAN_MODE_BRIDGE,
}
if err := netlink.LinkAdd(macvlandev); err != nil {
log.Fatalf("failed to create Macvlan: [ %v ] with the error: %s", macvlandev.Attrs().Name, err)
}
// log.Infof("Created Macvlan port: [ %s ] using the mode: [ %s ]", macvlan.Name, macvlanMode)
// ugly, actually ,can get the ns from netns.getfromDocker. the netns have many function, netns.getformpid
// netns.getfromdocker the arg can not be the container name
dockerPid := utils.DockerPid(CliCName)
//the macvlandev can be use directly, don't get netlink.byname again.
netlink.LinkSetNsPid(macvlandev, dockerPid)
runtime.LockOSThread()
defer runtime.UnlockOSThread()
//get root network namespace
origns, _ := netns.Get()
defer origns.Close()
//enter the docker container network
dockerNS, _ := netns.GetFromPid(dockerPid)
defer dockerNS.Close()
netns.Set(dockerNS)
// use macvlandev can cause error,need type assertion. netlink.Macvlan not must be netlink.Link,fmz(realy? the vlan action add)
// it is wrong,
macvlandev1, _ := netlink.LinkByName(macvlandev.Attrs().Name)
// when the eth is up, set name fail,: Device or resource busy
netlink.LinkSetDown(macvlandev1)
netlink.LinkSetName(macvlandev1, "eth1")
addr, err := netlink.ParseAddr(CliIP)
if err != nil {
log.Fatalf("failed to parse the ip address %v", CliIP)
}
netlink.AddrAdd(macvlandev1, addr)
netlink.LinkSetUp(macvlandev1)
/* set the default route, have some problem. Dst == 0.0.0.0/0? no
defaultgw := &netlink.Route{
Dst: nil,
}
netlink.RouteDel(defaultgw)
ip, _ := net.ParseIP("8.8.8.8")
routes, _ := netlink.RouteGet(ip)
for _, r := range routes {
netlink.RouteDel(&r)
}
*/
//if use ip instruction, it also can config the container, --privileged have no effect.
// The sublime test code(test this function) is strange, it only can avaiable in first time. And then fail(even need to reboot)
// got it,
//following code successfully delete the default route in docker container,but error in my host ,no such process
routes, _ := netlink.RouteList(nil, netlink.FAMILY_V4)
for _, r := range routes {
if r.Dst == nil {
if err := netlink.RouteDel(&r); err != nil {
log.Warnf("delete the default error: ", err)
}
}
}
if CligwIP == "" {
log.Fatal("container gw is null")
}
defaultRoute := &netlink.Route{
Dst: nil,
Gw: net.ParseIP(CligwIP),
LinkIndex: macvlandev1.Attrs().Index,
}
if err := netlink.RouteAdd(defaultRoute); err != nil {
log.Warnf("create default route error: ", err)
}
netns.Set(origns)
}
func createTupperware(i int, uri *url.URL, docker *dockerclient.DockerClient) {
defer wg.Done()
logrus.Infof("Giving tupperware container %d some spears", i)
// create the command flags to pass to ab
cmd := []string{
"ab",
"-c",
strconv.Itoa(concurrency),
"-n",
strconv.Itoa(requests),
"-m",
strings.ToUpper(method),
"-s",
strconv.Itoa(timeout),
"-v",
strconv.Itoa(verbosity),
"-f",
protocol,
}
if authHeader != "" {
cmd = append(cmd, []string{"-A", authHeader}...)
}
if proxyAuth != "" {
cmd = append(cmd, []string{"-P", proxyAuth}...)
}
if contentType != "" {
cmd = append(cmd, []string{"-T", contentType}...)
}
if timelimit > 0 {
cmd = append(cmd, []string{"-t", strconv.Itoa(timelimit)}...)
}
if len(headers) > 0 {
for _, header := range headers {
cmd = append(cmd, []string{"-H", header}...)
}
}
if len(cookies) > 0 {
for _, cookie := range cookies {
cmd = append(cmd, []string{"-C", cookie}...)
}
}
// append the uri to the cmd string
// make sure there is a trailing slash if none given
if uri.Path == "" {
uri.Path = "/"
}
cmd = append(cmd, uri.String())
// create the container
containerConfig := &dockerclient.ContainerConfig{
Image: "jess/ab",
Entrypoint: []string{"top"},
}
name := fmt.Sprintf("tws_%d", i)
id, err := docker.CreateContainer(containerConfig, name)
if err != nil {
logrus.Errorf("Error while creating container (%s): %v", name, err)
return
}
containers = append(containers, id)
// start the container
hostConfig := &dockerclient.HostConfig{}
if err = docker.StartContainer(id, hostConfig); err != nil {
logrus.Errorf("Error while starting container (%s): %v", name, err)
return
}
// we have to start the container _before_ adding the new default gateway
// for outbound traffic, its unfortunate but yeah we need the pid of the process
if cidr != "" {
// get the pid of the container
info, err := docker.InspectContainer(id)
if err != nil {
logrus.Errorf("Error while inspecting container (%s): %v", name, err)
return
}
pid := info.State.Pid
nsPidPath := path.Join(netnsPath, strconv.Itoa(pid))
// defer removal of the pid from /var/run/netns
defer os.RemoveAll(nsPidPath)
// create a symlink from proc to the netns pid
procPidPath := path.Join("/proc", strconv.Itoa(pid), "ns", "net")
if err := os.Symlink(procPidPath, nsPidPath); err != nil {
logrus.Errorf("could not create symlink from %s to %s: %v", procPidPath, nsPidPath, err)
}
// create the veth pair and add to bridge
local, guest, err := ovs.CreateVethPair(bridge)
if err != nil {
logrus.Error(err)
return
}
// get the local link
localLink, err := netlink.LinkByName(local)
if err != nil {
logrus.Errorf("getting link by name %s failed: %v", local, err)
return
}
// set the local link as up
if netlink.LinkSetUp(localLink); err != nil {
logrus.Errorf("setting link name %s as up failed: %v", local, err)
return
}
// get the guest link and setns as container pid
guestLink, err := netlink.LinkByName(guest)
if err != nil {
logrus.Errorf("getting link by name %s failed: %v", guest, err)
return
}
if err := netlink.LinkSetNsPid(guestLink, pid); err != nil {
logrus.Errorf("setting link name %s to netns pid %d failed: %v", guest, pid, err)
return
}
// set the interface to eth1 in the container
ciface := "eth1"
if _, err := ovs.NetNSExec(pid, "ip", "link", "set", guest, "name", ciface); err != nil {
logrus.Error(err)
return
}
// add the ip to the interface
if _, err := ovs.NetNSExec(pid, "ip", "addr", "add", ip.String(), "dev", ciface); err != nil {
logrus.Error(err)
return
}
// delete the default route
if _, err := ovs.NetNSExec(pid, "ip", "route", "delete", "default"); err != nil {
logrus.Warn(err)
}
// setup the gateway
if _, err := ovs.NetNSExec(pid, "ip", "route", "get", gateway); err != nil {
// add it
if _, err := ovs.NetNSExec(pid, "ip", "route", "add", fmt.Sprintf("%s/32", gateway), "dev", ciface); err != nil {
logrus.Error(err)
return
}
}
// set gateway as default
if _, err := ovs.NetNSExec(pid, "ip", "route", "replace", "default", "via", gateway); err != nil {
logrus.Error(err)
return
}
}
// exec ab in the container
args := append([]string{"exec", id}, cmd...)
output, err := exec.Command(dockerPath, args...).CombinedOutput()
if err != nil {
logrus.Errorf("docker exec (%s) failed: %v: %s (%s)", id[0:7], strings.Join(args, " "), output, err)
return
}
logrus.Infof("Output from container (%s)\n %s", name, output)
}
func AddVlannetwork(etcdval string, vlanid string, containerName string) {
ss := strings.Split(etcdval, ",")
hostif := ss[0]
if ok := utils.ValidateHostIface(hostif); !ok {
log.Warnf("the host interface not exist")
return
}
vlandevName := hostif + "." + vlanid
hostEth, _ := netlink.LinkByName(hostif)
intvlanid, err := strconv.Atoi(vlanid)
if err != nil {
log.Warnf("the vlan id convert error: \n")
return
}
var vlandev *netlink.Vlan
if ok := utils.ValidateHostIface(vlandevName); ok {
} else {
//not exist ,create the vlan device
vlandev = &netlink.Vlan{
LinkAttrs: netlink.LinkAttrs{
Name: vlandevName,
ParentIndex: hostEth.Attrs().Index,
},
VlanId: intvlanid,
}
if err := netlink.LinkAdd(vlandev); err != nil {
log.Warnf("failed to create vlandev: [ %v ] with the error: %s", vlandev, err)
return
}
}
netlink.LinkSetUp(vlandev)
macvlanname, _ := utils.GenerateRandomName("vlan"+vlanid, 5)
//create the macvlan device
macvlandev := &netlink.Macvlan{
LinkAttrs: netlink.LinkAttrs{
Name: macvlanname,
ParentIndex: vlandev.Attrs().Index,
},
Mode: netlink.MACVLAN_MODE_BRIDGE,
}
if err := netlink.LinkAdd(macvlandev); err != nil {
log.Warnf("failed to create Macvlan: [ %v ] with the error: %s", macvlandev, err)
return
}
dockerPid := utils.DockerPid(containerName)
//the macvlandev can be use directly, don't get netlink.byname again.
netlink.LinkSetNsPid(macvlandev, dockerPid)
runtime.LockOSThread()
defer runtime.UnlockOSThread()
//get root network naAddVlannetworkmespace
origns, _ := netns.Get()
defer origns.Close()
//enter the docker container network
dockerNS, _ := netns.GetFromPid(dockerPid)
defer dockerNS.Close()
netns.Set(dockerNS)
netlink.LinkSetDown(macvlandev)
netlink.LinkSetName(macvlandev, "eth1")
_, network, _ := net.ParseCIDR(ss[1])
if _, ok := ipallocs[vlanid]; !ok {
log.Fatalf("the ipallocator is null \n")
}
ip, _ := ipallocs[vlanid].RequestIP(network, nil)
ind := strings.LastIndex(ss[1], "/")
ipstring := ip.String() + ss[1][ind:]
addr, err := netlink.ParseAddr(ipstring)
netlink.AddrAdd(macvlandev, addr)
netlink.LinkSetUp(macvlandev)
/*
routes, _ := netlink.RouteList(nil, netlink.FAMILY_V4)
for _, r := range routes {
if r.Dst == nil {
if err := netlink.RouteDel(&r); err != nil {
log.Warnf("delete the default error: ", err)
}
}
}
if CligwIP == "" {
log.Fatal("container gw is null")
}
defaultRoute := &netlink.Route{
Dst: nil,
Gw: net.ParseIP(CligwIP),
LinkIndex: macvlandev1.Attrs().Index,
}
if err := netlink.RouteAdd(defaultRoute); err != nil {
log.Warnf("create default route error: ", err)
}
*/
netns.Set(origns)
}
