func (plugin *ovsPlugin) DeleteHostSubnetRules(subnet *osapi.HostSubnet) { glog.V(5).Infof("DeleteHostSubnetRules for %v", subnet) otx := ovs.NewTransaction(BR) otx.DeleteFlows("cookie=0x%s/0xffffffff", generateCookie(subnet.HostIP)) err := otx.EndTransaction() if err != nil { glog.Errorf("Error deleting OVS flows: %v", err) } }
func (plugin *ovsPlugin) DeleteHostSubnetRules(subnet *osapi.HostSubnet) error { glog.Infof("DeleteHostSubnetRules for %s", osdn.HostSubnetToString(subnet)) otx := ovs.NewTransaction(BR) otx.DeleteFlows("table=1, tun_src=%s", subnet.HostIP) otx.DeleteFlows("table=8, nw_dst=%s", subnet.Subnet) err := otx.EndTransaction() if err != nil { return fmt.Errorf("Error deleting OVS flows for subnet: %v, %v", subnet, err) } return nil }
func alreadySetUp(multitenant bool, localSubnetGatewayCIDR string) bool { var found bool itx := ipcmd.NewTransaction(LBR) addrs, err := itx.GetAddresses() itx.EndTransaction() if err != nil { return false } found = false for _, addr := range addrs { if addr == localSubnetGatewayCIDR { found = true break } } if !found { return false } otx := ovs.NewTransaction(BR) flows, err := otx.DumpFlows() otx.EndTransaction() if err != nil { return false } found = false for _, flow := range flows { if !strings.Contains(flow, VERSION_TABLE) { continue } idx := strings.Index(flow, VERSION_ACTION) if idx < 0 { continue } // OVS note action format hex bytes separated by '.'; first // byte is plugin type (multi-tenant/single-tenant) and second // byte is flow rule version expected := getPluginVersion(multitenant) existing := strings.Split(flow[idx+len(VERSION_ACTION):], ".") if len(existing) >= 2 && existing[0] == expected[0] && existing[1] == expected[1] { found = true break } } if !found { return false } return true }
func (plugin *ovsPlugin) AddHostSubnetRules(subnet *osapi.HostSubnet) { glog.V(5).Infof("AddHostSubnetRules for %v", subnet) cookie := generateCookie(subnet.HostIP) otx := ovs.NewTransaction(BR) otx.AddFlow("table=1, cookie=0x%s, priority=100, tun_src=%s, actions=goto_table:5", cookie, subnet.HostIP) otx.AddFlow("table=8, cookie=0x%s, priority=100, arp, nw_dst=%s, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:%s->tun_dst,output:1", cookie, subnet.Subnet, subnet.HostIP) otx.AddFlow("table=8, cookie=0x%s, priority=100, ip, nw_dst=%s, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:%s->tun_dst,output:1", cookie, subnet.Subnet, subnet.HostIP) err := otx.EndTransaction() if err != nil { glog.Errorf("Error adding OVS flows: %v", err) } }
func (plugin *ovsPlugin) AddHostSubnetRules(subnet *osapi.HostSubnet) error { glog.Infof("AddHostSubnetRules for %s", osdn.HostSubnetToString(subnet)) otx := ovs.NewTransaction(BR) otx.AddFlow("table=1, priority=100, tun_src=%s, actions=goto_table:5", subnet.HostIP) otx.AddFlow("table=8, priority=100, arp, nw_dst=%s, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:%s->tun_dst,output:1", subnet.Subnet, subnet.HostIP) otx.AddFlow("table=8, priority=100, ip, nw_dst=%s, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:%s->tun_dst,output:1", subnet.Subnet, subnet.HostIP) err := otx.EndTransaction() if err != nil { return fmt.Errorf("Error adding OVS flows for subnet: %v, %v", subnet, err) } return nil }
func (c *FlowController) DelServiceOFRules(netID uint, IP string, protocol api.ServiceProtocol, port uint) error { if !c.multitenant { return nil } glog.V(5).Infof("DelServiceOFRules for %s/%s/%d", IP, string(protocol), port) otx := ovs.NewTransaction(BR) otx.DeleteFlows(generateDelServiceRule(IP, protocol, port)) err := otx.EndTransaction() if err != nil { glog.Errorf("Error deleting OVS flow: %v", err) } return err }
func (c *FlowController) DelOFRules(nodeIP, localIP string) error { if nodeIP == localIP { return nil } glog.V(5).Infof("DelOFRules for %s", nodeIP) otx := ovs.NewTransaction(BR) otx.DeleteFlows("cookie=0x%s/0xffffffff", generateCookie(nodeIP)) err := otx.EndTransaction() if err != nil { glog.Errorf("Error deleting OVS flows: %v", err) } return err }
func (plugin *ovsPlugin) DeleteServiceRules(service *kapi.Service) { if !plugin.multitenant { return } glog.V(5).Infof("DeleteServiceRules for %v", service) otx := ovs.NewTransaction(BR) for _, port := range service.Spec.Ports { otx.DeleteFlows(generateDeleteServiceRule(service.Spec.ClusterIP, port.Protocol, port.Port)) err := otx.EndTransaction() if err != nil { glog.Errorf("Error deleting OVS flow: %v", err) } } }
func (plugin *ovsPlugin) AddServiceRules(service *kapi.Service, netID uint) { if !plugin.multitenant { return } glog.V(5).Infof("AddServiceRules for %v", service) otx := ovs.NewTransaction(BR) for _, port := range service.Spec.Ports { otx.AddFlow(generateAddServiceRule(netID, service.Spec.ClusterIP, port.Protocol, port.Port)) err := otx.EndTransaction() if err != nil { glog.Errorf("Error adding OVS flow: %v", err) } } }
func (plugin *OsdnNode) DeleteServiceRules(service *kapi.Service) error { if !plugin.multitenant { return nil } glog.V(5).Infof("DeleteServiceRules for %v", service) otx := ovs.NewTransaction(BR) for _, port := range service.Spec.Ports { otx.DeleteFlows(generateDeleteServiceRule(service.Spec.ClusterIP, port.Protocol, int(port.Port))) err := otx.EndTransaction() if err != nil { return fmt.Errorf("Error deleting OVS flows for service: %v, %v", service, err) } } return nil }
func (plugin *OsdnNode) AddServiceRules(service *kapi.Service, netID uint) error { if !plugin.multitenant { return nil } glog.V(5).Infof("AddServiceRules for %v", service) otx := ovs.NewTransaction(BR) for _, port := range service.Spec.Ports { otx.AddFlow(generateAddServiceRule(netID, service.Spec.ClusterIP, port.Protocol, int(port.Port))) err := otx.EndTransaction() if err != nil { return fmt.Errorf("Error adding OVS flows for service: %v, netid: %d, %v", service, netID, err) } } return nil }
func alreadySetUp(multitenant bool, localSubnetGatewayCIDR string) bool { var found bool itx := ipcmd.NewTransaction(LBR) addrs, err := itx.GetAddresses() itx.EndTransaction() if err != nil { return false } found = false for _, addr := range addrs { if addr == localSubnetGatewayCIDR { found = true break } } if !found { return false } otx := ovs.NewTransaction(BR) flows, err := otx.DumpFlows() otx.EndTransaction() if err != nil { return false } found = false for _, flow := range flows { if !strings.Contains(flow, "table=3") { continue } if (multitenant && strings.Contains(flow, "NXM_NX_TUN_ID")) || (!multitenant && strings.Contains(flow, "goto_table:9")) { found = true break } } if !found { return false } return true }
func (c *FlowController) AddOFRules(nodeIP, nodeSubnetCIDR, localIP string) error { if nodeIP == localIP { return nil } glog.V(5).Infof("AddOFRules for %s", nodeIP) cookie := generateCookie(nodeIP) otx := ovs.NewTransaction(BR) otx.AddFlow("table=0,cookie=0x%s,tun_src=%s,actions=goto_table:1", cookie, nodeIP) otx.AddFlow("table=8,cookie=0x%s,priority=100,ip,nw_dst=%s,actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:%s->tun_dst,output:1", cookie, nodeSubnetCIDR, nodeIP) otx.AddFlow("table=9,cookie=0x%s,priority=100,arp,nw_dst=%s,actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:%s->tun_dst,output:1", cookie, nodeSubnetCIDR, nodeIP) err := otx.EndTransaction() if err != nil { glog.Errorf("Error adding OVS flows: %v", err) } return err }
func (c *FlowController) Setup(localSubnetCIDR, clusterNetworkCIDR, servicesNetworkCIDR string, mtu uint) (bool, error) { _, ipnet, err := net.ParseCIDR(localSubnetCIDR) localSubnetMaskLength, _ := ipnet.Mask.Size() localSubnetGateway := netutils.GenerateDefaultGateway(ipnet).String() glog.V(5).Infof("[SDN setup] node pod subnet %s gateway %s", ipnet.String(), localSubnetGateway) gwCIDR := fmt.Sprintf("%s/%d", localSubnetGateway, localSubnetMaskLength) if alreadySetUp(c.multitenant, gwCIDR) { glog.V(5).Infof("[SDN setup] no SDN setup required") return false, nil } glog.V(5).Infof("[SDN setup] full SDN setup required") mtuStr := fmt.Sprint(mtu) itx := ipcmd.NewTransaction(LBR) itx.SetLink("down") itx.IgnoreError() itx.DeleteLink() itx.IgnoreError() itx.AddLink("type", "bridge") itx.AddAddress(gwCIDR) itx.SetLink("up") err = itx.EndTransaction() if err != nil { glog.Errorf("Failed to configure docker bridge: %v", err) return false, err } defer deleteLocalSubnetRoute(LBR, localSubnetCIDR) glog.V(5).Infof("[SDN setup] docker setup %s mtu %s", LBR, mtuStr) out, err := exec.Command("openshift-sdn-docker-setup.sh", LBR, mtuStr).CombinedOutput() if err != nil { glog.Errorf("Failed to configure docker networking: %v\n%s", err, out) return false, err } else { glog.V(5).Infof("[SDN setup] docker setup success:\n%s", out) } config := fmt.Sprintf("export OPENSHIFT_CLUSTER_SUBNET=%s", clusterNetworkCIDR) err = ioutil.WriteFile("/run/openshift-sdn/config.env", []byte(config), 0644) if err != nil { return false, err } itx = ipcmd.NewTransaction(VLINUXBR) itx.DeleteLink() itx.IgnoreError() itx.AddLink("mtu", mtuStr, "type", "veth", "peer", "name", VOVSBR, "mtu", mtuStr) itx.SetLink("up") itx.SetLink("txqueuelen", "0") err = itx.EndTransaction() if err != nil { return false, err } itx = ipcmd.NewTransaction(VOVSBR) itx.SetLink("up") itx.SetLink("txqueuelen", "0") err = itx.EndTransaction() if err != nil { return false, err } itx = ipcmd.NewTransaction(LBR) itx.AddSlave(VLINUXBR) err = itx.EndTransaction() if err != nil { return false, err } otx := ovs.NewTransaction(BR) otx.AddBridge("fail-mode=secure", "protocols=OpenFlow13") otx.AddPort(VXLAN, 1, "type=vxlan", `options:remote_ip="flow"`, `options:key="flow"`) otx.AddPort(TUN, 2, "type=internal") otx.AddPort(VOVSBR, 3) // Table 0; VXLAN filtering; the first rule sends un-tunnelled packets // to table 1. Additional per-node rules are filled in by controller.go otx.AddFlow("table=0, tun_src=0.0.0.0, actions=goto_table:1") // eg, "table=0, tun_src=${remote_node}, actions=goto_table:1" // Table 1; learn MAC addresses and continue with table 2 otx.AddFlow("table=1, actions=learn(table=9, priority=200, hard_timeout=900, NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[], load:NXM_NX_TUN_IPV4_SRC[]->NXM_NX_TUN_IPV4_DST[], output:NXM_OF_IN_PORT[]), goto_table:2") // Table 2; initial dispatch otx.AddFlow("table=2, priority=200, arp, actions=goto_table:9") otx.AddFlow("table=2, priority=100, in_port=1, actions=goto_table:3") // vxlan0 otx.AddFlow("table=2, priority=100, in_port=2, actions=goto_table:6") // tun0 otx.AddFlow("table=2, priority=100, in_port=3, actions=goto_table:6") // vovsbr otx.AddFlow("table=2, priority=0, actions=goto_table:4") // container // Table 3; incoming from vxlan otx.AddFlow("table=3, priority=200, ip, nw_dst=%s, actions=output:2", localSubnetGateway) if c.multitenant { otx.AddFlow("table=3, priority=100, ip, nw_dst=%s, actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[], goto_table:6", localSubnetCIDR) } else { otx.AddFlow("table=3, priority=100, ip, nw_dst=%s, actions=goto_table:9", localSubnetCIDR) } // Table 4; incoming from container; filled in by openshift-sdn-ovs // eg, single-tenant: "table=4, priority=100, in_port=${ovs_port}, ip, nw_src=${ipaddr}, goto_table:6" // multitenant: "table=4, priority=100, in_port=${ovs_port}, ip, nw_src=${ipaddr}, actions=load:${tenant_id}->NXM_NX_REG0[], goto_table:5" // Table 5; service isolation; mostly filled in by AddServiceOFRules() if c.multitenant { otx.AddFlow("table=5, priority=200, reg0=0, ip, nw_dst=%s, actions=output:2", servicesNetworkCIDR) // eg, "table=5, priority=200, ${service_proto}, nw_dst=${service_ip}, tp_dst=${service_port}, actions=output:2" otx.AddFlow("table=5, priority=100, ip, nw_dst=%s, actions=drop", servicesNetworkCIDR) otx.AddFlow("table=5, priority=0, actions=goto_table:6") } // Table 6; general routing otx.AddFlow("table=6, priority=200, ip, nw_dst=%s, actions=output:2", localSubnetGateway) if c.multitenant { otx.AddFlow("table=6, priority=175, ip, reg0=0, nw_dst=%s, actions=goto_table:9", localSubnetCIDR) otx.AddFlow("table=6, priority=150, ip, nw_dst=%s, actions=goto_table:7", localSubnetCIDR) } else { otx.AddFlow("table=6, priority=150, ip, nw_dst=%s, actions=goto_table:9", localSubnetCIDR) } otx.AddFlow("table=6, priority=100, ip, nw_dst=%s, actions=goto_table:8", clusterNetworkCIDR) otx.AddFlow("table=6, priority=0, ip, actions=output:2") // Table 7; to local container with isolation; filled in by openshift-sdn-ovs // eg, "table=7, priority=100, ip, nw_dst=${ipaddr}, reg0=${tenant_id}, actions=output:${ovs_port}" // Table 8; to remote container; filled in by AddOFRules() // eg, "table=8, priority=100, ip, nw_dst=${remote_subnet_cidr}, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31], set_field:${remote_node_ip}->tun_dst,output:1" // Table 9; MAC dispatch / ARP, filled in by Table 1's learn() rule // and with per-node vxlan ARP rules by AddOFRules() otx.AddFlow("table=9, priority=0, arp, actions=flood") // eg, "table=9, priority=100, arp, nw_dst=${remote_subnet_cidr}, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31], set_field:${remote_node_ip}->tun_dst,output:1" err = otx.EndTransaction() if err != nil { return false, err } itx = ipcmd.NewTransaction(TUN) itx.AddAddress(gwCIDR) defer deleteLocalSubnetRoute(TUN, localSubnetCIDR) itx.SetLink("mtu", mtuStr) itx.SetLink("up") itx.AddRoute(clusterNetworkCIDR, "proto", "kernel", "scope", "link") itx.AddRoute(servicesNetworkCIDR) err = itx.EndTransaction() if err != nil { return false, err } // Clean up docker0 since docker won't itx = ipcmd.NewTransaction("docker0") itx.SetLink("down") itx.IgnoreError() itx.DeleteLink() itx.IgnoreError() _ = itx.EndTransaction() // Disable iptables for linux bridges (and in particular lbr0), ignoring errors. // (This has to have been performed in advance for docker-in-docker deployments, // since this will fail there). _, _ = exec.Command("modprobe", "br_netfilter").CombinedOutput() err = sysctl.SetSysctl("net/bridge/bridge-nf-call-iptables", 0) if err != nil { glog.Warningf("Could not set net.bridge.bridge-nf-call-iptables sysctl: %s", err) } else { glog.V(5).Infof("[SDN setup] set net.bridge.bridge-nf-call-iptables to 0") } // Enable IP forwarding for ipv4 packets err = sysctl.SetSysctl("net/ipv4/ip_forward", 1) if err != nil { return false, fmt.Errorf("Could not enable IPv4 forwarding: %s", err) } err = sysctl.SetSysctl(fmt.Sprintf("net/ipv4/conf/%s/forwarding", TUN), 1) if err != nil { return false, fmt.Errorf("Could not enable IPv4 forwarding on %s: %s", TUN, err) } return true, nil }
func (plugin *ovsPlugin) SetupSDN(localSubnetCIDR, clusterNetworkCIDR, servicesNetworkCIDR string, mtu uint) (bool, error) { _, ipnet, err := net.ParseCIDR(localSubnetCIDR) localSubnetMaskLength, _ := ipnet.Mask.Size() localSubnetGateway := netutils.GenerateDefaultGateway(ipnet).String() glog.V(5).Infof("[SDN setup] node pod subnet %s gateway %s", ipnet.String(), localSubnetGateway) gwCIDR := fmt.Sprintf("%s/%d", localSubnetGateway, localSubnetMaskLength) if alreadySetUp(plugin.multitenant, gwCIDR) { glog.V(5).Infof("[SDN setup] no SDN setup required") return false, nil } glog.V(5).Infof("[SDN setup] full SDN setup required") mtuStr := fmt.Sprint(mtu) itx := ipcmd.NewTransaction(LBR) itx.SetLink("down") itx.IgnoreError() itx.DeleteLink() itx.IgnoreError() itx.AddLink("type", "bridge") itx.AddAddress(gwCIDR) itx.SetLink("up") err = itx.EndTransaction() if err != nil { glog.Errorf("Failed to configure docker bridge: %v", err) return false, err } defer deleteLocalSubnetRoute(LBR, localSubnetCIDR) glog.V(5).Infof("[SDN setup] docker setup %s mtu %s", LBR, mtuStr) out, err := exec.Command("openshift-sdn-docker-setup.sh", LBR, mtuStr).CombinedOutput() if err != nil { glog.Errorf("Failed to configure docker networking: %v\n%s", err, out) return false, err } else { glog.V(5).Infof("[SDN setup] docker setup success:\n%s", out) } config := fmt.Sprintf("export OPENSHIFT_CLUSTER_SUBNET=%s", clusterNetworkCIDR) err = ioutil.WriteFile("/run/openshift-sdn/config.env", []byte(config), 0644) if err != nil { return false, err } itx = ipcmd.NewTransaction(VLINUXBR) itx.DeleteLink() itx.IgnoreError() itx.AddLink("mtu", mtuStr, "type", "veth", "peer", "name", VOVSBR, "mtu", mtuStr) itx.SetLink("up") itx.SetLink("txqueuelen", "0") err = itx.EndTransaction() if err != nil { return false, err } itx = ipcmd.NewTransaction(VOVSBR) itx.SetLink("up") itx.SetLink("txqueuelen", "0") err = itx.EndTransaction() if err != nil { return false, err } itx = ipcmd.NewTransaction(LBR) itx.AddSlave(VLINUXBR) err = itx.EndTransaction() if err != nil { return false, err } otx := ovs.NewTransaction(BR) otx.AddBridge("fail-mode=secure", "protocols=OpenFlow13") otx.AddPort(VXLAN, 1, "type=vxlan", `options:remote_ip="flow"`, `options:key="flow"`) otx.AddPort(TUN, 2, "type=internal") otx.AddPort(VOVSBR, 3) // Table 0: initial dispatch based on in_port // vxlan0 otx.AddFlow("table=0, priority=200, in_port=1, arp, nw_src=%s, nw_dst=%s, actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:1", clusterNetworkCIDR, localSubnetCIDR) otx.AddFlow("table=0, priority=200, in_port=1, ip, nw_src=%s, nw_dst=%s, actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:1", clusterNetworkCIDR, localSubnetCIDR) otx.AddFlow("table=0, priority=150, in_port=1, actions=drop") // tun0 otx.AddFlow("table=0, priority=200, in_port=2, arp, nw_src=%s, nw_dst=%s, actions=goto_table:5", localSubnetGateway, clusterNetworkCIDR) otx.AddFlow("table=0, priority=200, in_port=2, ip, actions=goto_table:5") otx.AddFlow("table=0, priority=150, in_port=2, actions=drop") // vovsbr otx.AddFlow("table=0, priority=200, in_port=3, arp, nw_src=%s, actions=goto_table:5", localSubnetCIDR) otx.AddFlow("table=0, priority=200, in_port=3, ip, nw_src=%s, actions=goto_table:5", localSubnetCIDR) otx.AddFlow("table=0, priority=150, in_port=3, actions=drop") // else, from a container otx.AddFlow("table=0, priority=100, arp, actions=goto_table:2") otx.AddFlow("table=0, priority=100, ip, actions=goto_table:2") otx.AddFlow("table=0, priority=0, actions=drop") // Table 1: VXLAN ingress filtering; filled in by AddHostSubnetRules() // eg, "table=1, priority=100, tun_src=${remote_node_ip}, actions=goto_table:5" otx.AddFlow("table=1, priority=0, actions=drop") // Table 2: from OpenShift container; validate IP/MAC, assign tenant-id; filled in by openshift-sdn-ovs // eg, "table=2, priority=100, in_port=${ovs_port}, arp, nw_src=${ipaddr}, arp_sha=${macaddr}, actions=load:${tenant_id}->NXM_NX_REG0[], goto_table:5" // "table=2, priority=100, in_port=${ovs_port}, ip, nw_src=${ipaddr}, actions=load:${tenant_id}->NXM_NX_REG0[], goto_table:3" // (${tenant_id} is always 0 for single-tenant) otx.AddFlow("table=2, priority=0, actions=drop") // Table 3: from OpenShift container; service vs non-service otx.AddFlow("table=3, priority=100, ip, nw_dst=%s, actions=goto_table:4", servicesNetworkCIDR) otx.AddFlow("table=3, priority=0, actions=goto_table:5") // Table 4: from OpenShift container; service dispatch; filled in by AddServiceRules() otx.AddFlow("table=4, priority=200, reg0=0, actions=output:2") // eg, "table=4, priority=100, reg0=${tenant_id}, ${service_proto}, nw_dst=${service_ip}, tp_dst=${service_port}, actions=output:2" otx.AddFlow("table=4, priority=0, actions=drop") // Table 5: general routing otx.AddFlow("table=5, priority=300, arp, nw_dst=%s, actions=output:2", localSubnetGateway) otx.AddFlow("table=5, priority=300, ip, nw_dst=%s, actions=output:2", localSubnetGateway) otx.AddFlow("table=5, priority=200, arp, nw_dst=%s, actions=goto_table:6", localSubnetCIDR) otx.AddFlow("table=5, priority=200, ip, nw_dst=%s, actions=goto_table:7", localSubnetCIDR) otx.AddFlow("table=5, priority=100, arp, nw_dst=%s, actions=goto_table:8", clusterNetworkCIDR) otx.AddFlow("table=5, priority=100, ip, nw_dst=%s, actions=goto_table:8", clusterNetworkCIDR) otx.AddFlow("table=5, priority=0, ip, actions=output:2") otx.AddFlow("table=5, priority=0, arp, actions=drop") // Table 6: ARP to container, filled in by openshift-sdn-ovs // eg, "table=6, priority=100, arp, nw_dst=${container_ip}, actions=output:${ovs_port}" otx.AddFlow("table=6, priority=0, actions=output:3") // Table 7: IP to container; filled in by openshift-sdn-ovs // eg, "table=7, priority=100, reg0=0, ip, nw_dst=${ipaddr}, actions=output:${ovs_port}" // eg, "table=7, priority=100, reg0=${tenant_id}, ip, nw_dst=${ipaddr}, actions=output:${ovs_port}" otx.AddFlow("table=7, priority=0, actions=output:3") // Table 8: to remote container; filled in by AddHostSubnetRules() // eg, "table=8, priority=100, arp, nw_dst=${remote_subnet_cidr}, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31], set_field:${remote_node_ip}->tun_dst,output:1" // eg, "table=8, priority=100, ip, nw_dst=${remote_subnet_cidr}, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31], set_field:${remote_node_ip}->tun_dst,output:1" otx.AddFlow("table=8, priority=0, actions=drop") err = otx.EndTransaction() if err != nil { return false, err } itx = ipcmd.NewTransaction(TUN) itx.AddAddress(gwCIDR) defer deleteLocalSubnetRoute(TUN, localSubnetCIDR) itx.SetLink("mtu", mtuStr) itx.SetLink("up") itx.AddRoute(clusterNetworkCIDR, "proto", "kernel", "scope", "link") itx.AddRoute(servicesNetworkCIDR) err = itx.EndTransaction() if err != nil { return false, err } // Clean up docker0 since docker won't itx = ipcmd.NewTransaction("docker0") itx.SetLink("down") itx.IgnoreError() itx.DeleteLink() itx.IgnoreError() _ = itx.EndTransaction() // Disable iptables for linux bridges (and in particular lbr0), ignoring errors. // (This has to have been performed in advance for docker-in-docker deployments, // since this will fail there). _, _ = exec.Command("modprobe", "br_netfilter").CombinedOutput() err = sysctl.SetSysctl("net/bridge/bridge-nf-call-iptables", 0) if err != nil { glog.Warningf("Could not set net.bridge.bridge-nf-call-iptables sysctl: %s", err) } else { glog.V(5).Infof("[SDN setup] set net.bridge.bridge-nf-call-iptables to 0") } // Enable IP forwarding for ipv4 packets err = sysctl.SetSysctl("net/ipv4/ip_forward", 1) if err != nil { return false, fmt.Errorf("Could not enable IPv4 forwarding: %s", err) } err = sysctl.SetSysctl(fmt.Sprintf("net/ipv4/conf/%s/forwarding", TUN), 1) if err != nil { return false, fmt.Errorf("Could not enable IPv4 forwarding on %s: %s", TUN, err) } // Table 253: rule version; note action is hex bytes separated by '.' otx = ovs.NewTransaction(BR) pluginVersion := getPluginVersion(plugin.multitenant) otx.AddFlow("%s, %s%s.%s", VERSION_TABLE, VERSION_ACTION, pluginVersion[0], pluginVersion[1]) err = otx.EndTransaction() if err != nil { return false, err } return true, nil }