func deleteLocalSubnetRoute(device, localSubnetCIDR string) {
backoff := utilwait.Backoff{
Duration: 100 * time.Millisecond,
Factor: 1.25,
Steps: 6,
}
err := utilwait.ExponentialBackoff(backoff, func() (bool, error) {
itx := ipcmd.NewTransaction(kexec.New(), device)
routes, err := itx.GetRoutes()
if err != nil {
return false, fmt.Errorf("could not get routes: %v", err)
}
for _, route := range routes {
if strings.Contains(route, localSubnetCIDR) {
itx.DeleteRoute(localSubnetCIDR)
err = itx.EndTransaction()
if err != nil {
return false, fmt.Errorf("could not delete route: %v", err)
}
return true, nil
}
}
return false, nil
})
if err != nil {
glog.Errorf("Error removing %s route from dev %s: %v; if the route appears later it will not be deleted.", localSubnetCIDR, device, err)
}
}
// Detect whether we are upgrading from a pre-CNI openshift and clean up
// interfaces and iptables rules that are no longer required
func (node *OsdnNode) dockerPreCNICleanup() error {
exec := kexec.New()
itx := ipcmd.NewTransaction(exec, "lbr0")
itx.SetLink("down")
if err := itx.EndTransaction(); err != nil {
// no cleanup required
return nil
}
node.clearLbr0IptablesRule = true
// Restart docker to kill old pods and make it use docker0 again.
// "systemctl restart" will bail out (unnecessarily) in the
// OpenShift-in-a-container case, so we work around that by sending
// the messages by hand.
if _, err := osexec.Command("dbus-send", "--system", "--print-reply", "--reply-timeout=2000", "--type=method_call", "--dest=org.freedesktop.systemd1", "/org/freedesktop/systemd1", "org.freedesktop.systemd1.Manager.Reload").CombinedOutput(); err != nil {
log.Error(err)
}
if _, err := osexec.Command("dbus-send", "--system", "--print-reply", "--reply-timeout=2000", "--type=method_call", "--dest=org.freedesktop.systemd1", "/org/freedesktop/systemd1", "org.freedesktop.systemd1.Manager.RestartUnit", "string:'docker.service' string:'replace'").CombinedOutput(); err != nil {
log.Error(err)
}
// Delete pre-CNI interfaces
for _, intf := range []string{"lbr0", "vovsbr", "vlinuxbr"} {
itx := ipcmd.NewTransaction(exec, intf)
itx.DeleteLink()
itx.IgnoreError()
itx.EndTransaction()
}
// Wait until docker has restarted since kubelet will exit it docker isn't running
dockerClient, err := docker.NewClientFromEnv()
if err != nil {
return fmt.Errorf("failed to get docker client: %v", err)
}
err = kwait.ExponentialBackoff(
kwait.Backoff{
Duration: 100 * time.Millisecond,
Factor: 1.2,
Steps: 6,
},
func() (bool, error) {
if err := dockerClient.Ping(); err != nil {
// wait longer
return false, nil
}
return true, nil
})
if err != nil {
return fmt.Errorf("failed to connect to docker after SDN cleanup restart: %v", err)
}
log.Infof("Cleaned up left-over openshift-sdn docker bridge and interfaces")
return nil
}
func deleteLocalSubnetRoute(device, localSubnetCIDR string) {
const (
timeInterval = 100 * time.Millisecond
maxIntervals = 20
)
for i := 0; i < maxIntervals; i++ {
itx := ipcmd.NewTransaction(kexec.New(), device)
routes, err := itx.GetRoutes()
if err != nil {
glog.Errorf("Could not get routes for dev %s: %v", device, err)
return
}
for _, route := range routes {
if strings.Contains(route, localSubnetCIDR) {
itx.DeleteRoute(localSubnetCIDR)
err = itx.EndTransaction()
if err != nil {
glog.Errorf("Could not delete subnet route %s from dev %s: %v", localSubnetCIDR, device, err)
}
return
}
}
time.Sleep(timeInterval)
}
glog.Errorf("Timed out looking for %s route for dev %s; if it appears later it will not be deleted.", localSubnetCIDR, device)
}
func alreadySetUp(multitenant bool, localSubnetGatewayCIDR string) bool {
var found bool
exec := kexec.New()
itx := ipcmd.NewTransaction(exec, LBR)
addrs, err := itx.GetAddresses()
itx.EndTransaction()
if err != nil {
return false
}
found = false
for _, addr := range addrs {
if addr == localSubnetGatewayCIDR {
found = true
break
}
}
if !found {
return false
}
otx := ovs.NewTransaction(exec, BR)
flows, err := otx.DumpFlows()
otx.EndTransaction()
if err != nil {
return false
}
found = false
for _, flow := range flows {
if !strings.Contains(flow, VERSION_TABLE) {
continue
}
idx := strings.Index(flow, VERSION_ACTION)
if idx < 0 {
continue
}
// OVS note action format hex bytes separated by '.'; first
// byte is plugin type (multi-tenant/single-tenant) and second
// byte is flow rule version
expected := getPluginVersion(multitenant)
existing := strings.Split(flow[idx+len(VERSION_ACTION):], ".")
if len(existing) >= 2 && existing[0] == expected[0] && existing[1] == expected[1] {
found = true
break
}
}
if !found {
return false
}
return true
}
func (plugin *OsdnNode) alreadySetUp(localSubnetGatewayCIDR, clusterNetworkCIDR string) bool {
var found bool
exec := kexec.New()
itx := ipcmd.NewTransaction(exec, TUN)
addrs, err := itx.GetAddresses()
itx.EndTransaction()
if err != nil {
return false
}
found = false
for _, addr := range addrs {
if strings.Contains(addr, localSubnetGatewayCIDR+" ") {
found = true
break
}
}
if !found {
return false
}
itx = ipcmd.NewTransaction(exec, TUN)
routes, err := itx.GetRoutes()
itx.EndTransaction()
if err != nil {
return false
}
found = false
for _, route := range routes {
if strings.Contains(route, clusterNetworkCIDR+" ") {
found = true
break
}
}
if !found {
return false
}
flows, err := plugin.ovs.DumpFlows()
if err != nil {
return false
}
found = false
for _, flow := range flows {
if !strings.Contains(flow, VERSION_TABLE) {
continue
}
idx := strings.Index(flow, VERSION_ACTION)
if idx < 0 {
continue
}
// OVS note action format hex bytes separated by '.'; first
// byte is plugin type (multi-tenant/single-tenant) and second
// byte is flow rule version
expected := getPluginVersion(plugin.multitenant)
existing := strings.Split(flow[idx+len(VERSION_ACTION):], ".")
if len(existing) >= 2 && existing[0] == expected[0] && existing[1] == expected[1] {
found = true
break
}
}
if !found {
return false
}
return true
}
func (plugin *OsdnNode) SetupSDN() (bool, error) {
clusterNetworkCIDR := plugin.networkInfo.ClusterNetwork.String()
serviceNetworkCIDR := plugin.networkInfo.ServiceNetwork.String()
localSubnetCIDR := plugin.localSubnetCIDR
_, ipnet, err := net.ParseCIDR(localSubnetCIDR)
localSubnetMaskLength, _ := ipnet.Mask.Size()
localSubnetGateway := netutils.GenerateDefaultGateway(ipnet).String()
glog.V(5).Infof("[SDN setup] node pod subnet %s gateway %s", ipnet.String(), localSubnetGateway)
exec := kexec.New()
if plugin.clearLbr0IptablesRule {
// Delete docker's left-over lbr0 rule; cannot do this from
// NewNodePlugin (where docker is cleaned up) because we need
// localSubnetCIDR which is only valid after plugin start
ipt := iptables.New(exec, utildbus.New(), iptables.ProtocolIpv4)
ipt.DeleteRule(iptables.TableNAT, iptables.ChainPostrouting, "-s", localSubnetCIDR, "!", "-o", "lbr0", "-j", "MASQUERADE")
}
gwCIDR := fmt.Sprintf("%s/%d", localSubnetGateway, localSubnetMaskLength)
if plugin.alreadySetUp(gwCIDR, clusterNetworkCIDR) {
glog.V(5).Infof("[SDN setup] no SDN setup required")
return false, nil
}
glog.V(5).Infof("[SDN setup] full SDN setup required")
if err := os.MkdirAll("/run/openshift-sdn", 0700); err != nil {
return false, err
}
config := fmt.Sprintf("export OPENSHIFT_CLUSTER_SUBNET=%s", clusterNetworkCIDR)
err = ioutil.WriteFile("/run/openshift-sdn/config.env", []byte(config), 0644)
if err != nil {
return false, err
}
err = plugin.ovs.AddBridge("fail-mode=secure", "protocols=OpenFlow13")
if err != nil {
return false, err
}
_ = plugin.ovs.DeletePort(VXLAN)
_, err = plugin.ovs.AddPort(VXLAN, 1, "type=vxlan", `options:remote_ip="flow"`, `options:key="flow"`)
if err != nil {
return false, err
}
_ = plugin.ovs.DeletePort(TUN)
_, err = plugin.ovs.AddPort(TUN, 2, "type=internal")
if err != nil {
return false, err
}
otx := plugin.ovs.NewTransaction()
// Table 0: initial dispatch based on in_port
// vxlan0
otx.AddFlow("table=0, priority=200, in_port=1, arp, nw_src=%s, nw_dst=%s, actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:1", clusterNetworkCIDR, localSubnetCIDR)
otx.AddFlow("table=0, priority=200, in_port=1, ip, nw_src=%s, nw_dst=%s, actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:1", clusterNetworkCIDR, localSubnetCIDR)
otx.AddFlow("table=0, priority=150, in_port=1, actions=drop")
// tun0
otx.AddFlow("table=0, priority=200, in_port=2, arp, nw_src=%s, nw_dst=%s, actions=goto_table:5", localSubnetGateway, clusterNetworkCIDR)
otx.AddFlow("table=0, priority=200, in_port=2, ip, actions=goto_table:5")
otx.AddFlow("table=0, priority=150, in_port=2, actions=drop")
// else, from a container
otx.AddFlow("table=0, priority=100, arp, actions=goto_table:2")
otx.AddFlow("table=0, priority=100, ip, actions=goto_table:2")
otx.AddFlow("table=0, priority=0, actions=drop")
// Table 1: VXLAN ingress filtering; filled in by AddHostSubnetRules()
// eg, "table=1, priority=100, tun_src=${remote_node_ip}, actions=goto_table:5"
otx.AddFlow("table=1, priority=0, actions=drop")
// Table 2: from OpenShift container; validate IP/MAC, assign tenant-id; filled in by openshift-sdn-ovs
// eg, "table=2, priority=100, in_port=${ovs_port}, arp, nw_src=${ipaddr}, arp_sha=${macaddr}, actions=load:${tenant_id}->NXM_NX_REG0[], goto_table:5"
// "table=2, priority=100, in_port=${ovs_port}, ip, nw_src=${ipaddr}, actions=load:${tenant_id}->NXM_NX_REG0[], goto_table:3"
// (${tenant_id} is always 0 for single-tenant)
otx.AddFlow("table=2, priority=0, actions=drop")
// Table 3: from OpenShift container; service vs non-service
otx.AddFlow("table=3, priority=100, ip, nw_dst=%s, actions=goto_table:4", serviceNetworkCIDR)
otx.AddFlow("table=3, priority=0, actions=goto_table:5")
// Table 4: from OpenShift container; service dispatch; filled in by AddServiceRules()
otx.AddFlow("table=4, priority=200, reg0=0, actions=output:2")
// eg, "table=4, priority=100, reg0=${tenant_id}, ${service_proto}, nw_dst=${service_ip}, tp_dst=${service_port}, actions=output:2"
otx.AddFlow("table=4, priority=0, actions=drop")
// Table 5: general routing
otx.AddFlow("table=5, priority=300, arp, nw_dst=%s, actions=output:2", localSubnetGateway)
otx.AddFlow("table=5, priority=300, ip, nw_dst=%s, actions=output:2", localSubnetGateway)
otx.AddFlow("table=5, priority=200, arp, nw_dst=%s, actions=goto_table:6", localSubnetCIDR)
otx.AddFlow("table=5, priority=200, ip, nw_dst=%s, actions=goto_table:7", localSubnetCIDR)
otx.AddFlow("table=5, priority=100, arp, nw_dst=%s, actions=goto_table:8", clusterNetworkCIDR)
otx.AddFlow("table=5, priority=100, ip, nw_dst=%s, actions=goto_table:8", clusterNetworkCIDR)
otx.AddFlow("table=5, priority=0, ip, actions=goto_table:9")
otx.AddFlow("table=5, priority=0, arp, actions=drop")
// Table 6: ARP to container, filled in by openshift-sdn-ovs
// eg, "table=6, priority=100, arp, nw_dst=${container_ip}, actions=output:${ovs_port}"
otx.AddFlow("table=6, priority=0, actions=drop")
// Table 7: IP to container; filled in by openshift-sdn-ovs
// eg, "table=7, priority=100, reg0=0, ip, nw_dst=${ipaddr}, actions=output:${ovs_port}"
// eg, "table=7, priority=100, reg0=${tenant_id}, ip, nw_dst=${ipaddr}, actions=output:${ovs_port}"
otx.AddFlow("table=7, priority=0, actions=drop")
// Table 8: to remote container; filled in by AddHostSubnetRules()
// eg, "table=8, priority=100, arp, nw_dst=${remote_subnet_cidr}, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31], set_field:${remote_node_ip}->tun_dst,output:1"
// eg, "table=8, priority=100, ip, nw_dst=${remote_subnet_cidr}, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31], set_field:${remote_node_ip}->tun_dst,output:1"
otx.AddFlow("table=8, priority=0, actions=drop")
// Table 9: egress network policy dispatch; edited by updateEgressNetworkPolicyRules()
// eg, "table=9, reg0=${tenant_id}, priority=2, ip, nw_dst=${external_cidr}, actions=drop
otx.AddFlow("table=9, priority=0, actions=output:2")
err = otx.EndTransaction()
if err != nil {
return false, err
}
itx := ipcmd.NewTransaction(exec, TUN)
itx.AddAddress(gwCIDR)
defer deleteLocalSubnetRoute(TUN, localSubnetCIDR)
itx.SetLink("mtu", fmt.Sprint(plugin.mtu))
itx.SetLink("up")
itx.AddRoute(clusterNetworkCIDR, "proto", "kernel", "scope", "link")
itx.AddRoute(serviceNetworkCIDR)
err = itx.EndTransaction()
if err != nil {
return false, err
}
sysctl := sysctl.New()
// Enable IP forwarding for ipv4 packets
err = sysctl.SetSysctl("net/ipv4/ip_forward", 1)
if err != nil {
return false, fmt.Errorf("Could not enable IPv4 forwarding: %s", err)
}
err = sysctl.SetSysctl(fmt.Sprintf("net/ipv4/conf/%s/forwarding", TUN), 1)
if err != nil {
return false, fmt.Errorf("Could not enable IPv4 forwarding on %s: %s", TUN, err)
}
// Table 253: rule version; note action is hex bytes separated by '.'
otx = plugin.ovs.NewTransaction()
pluginVersion := getPluginVersion(plugin.multitenant)
otx.AddFlow("%s, %s%s.%s", VERSION_TABLE, VERSION_ACTION, pluginVersion[0], pluginVersion[1])
err = otx.EndTransaction()
if err != nil {
return false, err
}
return true, nil
}
func (plugin *OsdnNode) SetupSDN() (bool, error) {
localSubnetCIDR, err := plugin.getLocalSubnet()
if err != nil {
return false, err
}
clusterNetworkCIDR := plugin.networkInfo.ClusterNetwork.String()
serviceNetworkCIDR := plugin.networkInfo.ServiceNetwork.String()
_, ipnet, err := net.ParseCIDR(localSubnetCIDR)
localSubnetMaskLength, _ := ipnet.Mask.Size()
localSubnetGateway := netutils.GenerateDefaultGateway(ipnet).String()
glog.V(5).Infof("[SDN setup] node pod subnet %s gateway %s", ipnet.String(), localSubnetGateway)
gwCIDR := fmt.Sprintf("%s/%d", localSubnetGateway, localSubnetMaskLength)
if plugin.alreadySetUp(gwCIDR, clusterNetworkCIDR) {
glog.V(5).Infof("[SDN setup] no SDN setup required")
return false, nil
}
glog.V(5).Infof("[SDN setup] full SDN setup required")
mtuStr := fmt.Sprint(plugin.mtu)
exec := kexec.New()
itx := ipcmd.NewTransaction(exec, LBR)
itx.SetLink("down")
itx.IgnoreError()
itx.DeleteLink()
itx.IgnoreError()
itx.AddLink("type", "bridge")
itx.AddAddress(gwCIDR)
itx.SetLink("up")
err = itx.EndTransaction()
if err != nil {
glog.Errorf("Failed to configure docker bridge: %v", err)
return false, err
}
defer deleteLocalSubnetRoute(LBR, localSubnetCIDR)
glog.V(5).Infof("[SDN setup] docker setup %s mtu %s", LBR, mtuStr)
out, err := exec.Command("openshift-sdn-docker-setup.sh", LBR, mtuStr).CombinedOutput()
if err != nil {
glog.Errorf("Failed to configure docker networking: %v\n%s", err, out)
return false, err
} else {
glog.V(5).Infof("[SDN setup] docker setup success:\n%s", out)
}
config := fmt.Sprintf("export OPENSHIFT_CLUSTER_SUBNET=%s", clusterNetworkCIDR)
err = ioutil.WriteFile("/run/openshift-sdn/config.env", []byte(config), 0644)
if err != nil {
return false, err
}
itx = ipcmd.NewTransaction(exec, VLINUXBR)
itx.DeleteLink()
itx.IgnoreError()
itx.AddLink("mtu", mtuStr, "type", "veth", "peer", "name", VOVSBR, "mtu", mtuStr)
itx.SetLink("up")
itx.SetLink("txqueuelen", "0")
err = itx.EndTransaction()
if err != nil {
return false, err
}
itx = ipcmd.NewTransaction(exec, VOVSBR)
itx.SetLink("up")
itx.SetLink("txqueuelen", "0")
err = itx.EndTransaction()
if err != nil {
return false, err
}
itx = ipcmd.NewTransaction(exec, LBR)
itx.AddSlave(VLINUXBR)
err = itx.EndTransaction()
if err != nil {
return false, err
}
err = plugin.ovs.AddBridge("fail-mode=secure", "protocols=OpenFlow13")
if err != nil {
return false, err
}
_ = plugin.ovs.DeletePort(VXLAN)
_, err = plugin.ovs.AddPort(VXLAN, 1, "type=vxlan", `options:remote_ip="flow"`, `options:key="flow"`)
if err != nil {
return false, err
}
_ = plugin.ovs.DeletePort(TUN)
_, err = plugin.ovs.AddPort(TUN, 2, "type=internal")
if err != nil {
return false, err
}
_, err = plugin.ovs.AddPort(VOVSBR, 3)
if err != nil {
return false, err
}
otx := plugin.ovs.NewTransaction()
// Table 0: initial dispatch based on in_port
// vxlan0
otx.AddFlow("table=0, priority=200, in_port=1, arp, nw_src=%s, nw_dst=%s, actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:1", clusterNetworkCIDR, localSubnetCIDR)
otx.AddFlow("table=0, priority=200, in_port=1, ip, nw_src=%s, nw_dst=%s, actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:1", clusterNetworkCIDR, localSubnetCIDR)
otx.AddFlow("table=0, priority=150, in_port=1, actions=drop")
// tun0
otx.AddFlow("table=0, priority=200, in_port=2, arp, nw_src=%s, nw_dst=%s, actions=goto_table:5", localSubnetGateway, clusterNetworkCIDR)
otx.AddFlow("table=0, priority=200, in_port=2, ip, actions=goto_table:5")
otx.AddFlow("table=0, priority=150, in_port=2, actions=drop")
// vovsbr
otx.AddFlow("table=0, priority=200, in_port=3, arp, nw_src=%s, actions=goto_table:5", localSubnetCIDR)
otx.AddFlow("table=0, priority=200, in_port=3, ip, nw_src=%s, actions=goto_table:5", localSubnetCIDR)
otx.AddFlow("table=0, priority=150, in_port=3, actions=drop")
// else, from a container
otx.AddFlow("table=0, priority=100, arp, actions=goto_table:2")
otx.AddFlow("table=0, priority=100, ip, actions=goto_table:2")
otx.AddFlow("table=0, priority=0, actions=drop")
// Table 1: VXLAN ingress filtering; filled in by AddHostSubnetRules()
// eg, "table=1, priority=100, tun_src=${remote_node_ip}, actions=goto_table:5"
otx.AddFlow("table=1, priority=0, actions=drop")
// Table 2: from OpenShift container; validate IP/MAC, assign tenant-id; filled in by openshift-sdn-ovs
// eg, "table=2, priority=100, in_port=${ovs_port}, arp, nw_src=${ipaddr}, arp_sha=${macaddr}, actions=load:${tenant_id}->NXM_NX_REG0[], goto_table:5"
// "table=2, priority=100, in_port=${ovs_port}, ip, nw_src=${ipaddr}, actions=load:${tenant_id}->NXM_NX_REG0[], goto_table:3"
// (${tenant_id} is always 0 for single-tenant)
otx.AddFlow("table=2, priority=0, actions=drop")
// Table 3: from OpenShift container; service vs non-service
otx.AddFlow("table=3, priority=100, ip, nw_dst=%s, actions=goto_table:4", serviceNetworkCIDR)
otx.AddFlow("table=3, priority=0, actions=goto_table:5")
// Table 4: from OpenShift container; service dispatch; filled in by AddServiceRules()
otx.AddFlow("table=4, priority=200, reg0=0, actions=output:2")
// eg, "table=4, priority=100, reg0=${tenant_id}, ${service_proto}, nw_dst=${service_ip}, tp_dst=${service_port}, actions=output:2"
otx.AddFlow("table=4, priority=0, actions=drop")
// Table 5: general routing
otx.AddFlow("table=5, priority=300, arp, nw_dst=%s, actions=output:2", localSubnetGateway)
otx.AddFlow("table=5, priority=300, ip, nw_dst=%s, actions=output:2", localSubnetGateway)
otx.AddFlow("table=5, priority=200, arp, nw_dst=%s, actions=goto_table:6", localSubnetCIDR)
otx.AddFlow("table=5, priority=200, ip, nw_dst=%s, actions=goto_table:7", localSubnetCIDR)
otx.AddFlow("table=5, priority=100, arp, nw_dst=%s, actions=goto_table:8", clusterNetworkCIDR)
otx.AddFlow("table=5, priority=100, ip, nw_dst=%s, actions=goto_table:8", clusterNetworkCIDR)
otx.AddFlow("table=5, priority=0, ip, actions=goto_table:9")
otx.AddFlow("table=5, priority=0, arp, actions=drop")
// Table 6: ARP to container, filled in by openshift-sdn-ovs
// eg, "table=6, priority=100, arp, nw_dst=${container_ip}, actions=output:${ovs_port}"
otx.AddFlow("table=6, priority=0, actions=output:3")
// Table 7: IP to container; filled in by openshift-sdn-ovs
// eg, "table=7, priority=100, reg0=0, ip, nw_dst=${ipaddr}, actions=output:${ovs_port}"
// eg, "table=7, priority=100, reg0=${tenant_id}, ip, nw_dst=${ipaddr}, actions=output:${ovs_port}"
otx.AddFlow("table=7, priority=0, actions=output:3")
// Table 8: to remote container; filled in by AddHostSubnetRules()
// eg, "table=8, priority=100, arp, nw_dst=${remote_subnet_cidr}, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31], set_field:${remote_node_ip}->tun_dst,output:1"
// eg, "table=8, priority=100, ip, nw_dst=${remote_subnet_cidr}, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31], set_field:${remote_node_ip}->tun_dst,output:1"
otx.AddFlow("table=8, priority=0, actions=drop")
// Table 9: egress network policy dispatch; edited by updateEgressNetworkPolicyRules()
// eg, "table=9, reg0=${tenant_id}, priority=2, ip, nw_dst=${external_cidr}, actions=drop
otx.AddFlow("table=9, priority=0, actions=output:2")
err = otx.EndTransaction()
if err != nil {
return false, err
}
itx = ipcmd.NewTransaction(exec, TUN)
itx.AddAddress(gwCIDR)
defer deleteLocalSubnetRoute(TUN, localSubnetCIDR)
itx.SetLink("mtu", mtuStr)
itx.SetLink("up")
itx.AddRoute(clusterNetworkCIDR, "proto", "kernel", "scope", "link")
itx.AddRoute(serviceNetworkCIDR)
err = itx.EndTransaction()
if err != nil {
return false, err
}
// Clean up docker0 since docker won't
itx = ipcmd.NewTransaction(exec, "docker0")
itx.SetLink("down")
itx.IgnoreError()
itx.DeleteLink()
itx.IgnoreError()
_ = itx.EndTransaction()
sysctl := sysctl.New()
// Disable iptables for linux bridges (and in particular lbr0), ignoring errors.
// (This has to have been performed in advance for docker-in-docker deployments,
// since this will fail there).
_, _ = exec.Command("modprobe", "br_netfilter").CombinedOutput()
err = sysctl.SetSysctl("net/bridge/bridge-nf-call-iptables", 0)
if err != nil {
glog.Warningf("Could not set net.bridge.bridge-nf-call-iptables sysctl: %s", err)
} else {
glog.V(5).Infof("[SDN setup] set net.bridge.bridge-nf-call-iptables to 0")
}
// Enable IP forwarding for ipv4 packets
err = sysctl.SetSysctl("net/ipv4/ip_forward", 1)
if err != nil {
return false, fmt.Errorf("Could not enable IPv4 forwarding: %s", err)
}
err = sysctl.SetSysctl(fmt.Sprintf("net/ipv4/conf/%s/forwarding", TUN), 1)
if err != nil {
return false, fmt.Errorf("Could not enable IPv4 forwarding on %s: %s", TUN, err)
}
// Table 253: rule version; note action is hex bytes separated by '.'
otx = plugin.ovs.NewTransaction()
pluginVersion := getPluginVersion(plugin.multitenant)
otx.AddFlow("%s, %s%s.%s", VERSION_TABLE, VERSION_ACTION, pluginVersion[0], pluginVersion[1])
err = otx.EndTransaction()
if err != nil {
return false, err
}
return true, nil
}
