func SetupIptables(fw *firewalld.Interface, clusterNetworkCIDR string) error {
if fw.IsRunning() {
rules := []FirewallRule{
{firewalld.IPv4, "nat", "POSTROUTING", 0, []string{"-s", clusterNetworkCIDR, "!", "-d", clusterNetworkCIDR, "-j", "MASQUERADE"}},
{firewalld.IPv4, "filter", "INPUT", 0, []string{"-p", "udp", "-m", "multiport", "--dports", "4789", "-m", "comment", "--comment", "001 vxlan incoming", "-j", "ACCEPT"}},
{firewalld.IPv4, "filter", "INPUT", 0, []string{"-i", "tun0", "-m", "comment", "--comment", "traffic from docker for internet", "-j", "ACCEPT"}},
{firewalld.IPv4, "filter", "FORWARD", 0, []string{"-d", clusterNetworkCIDR, "-j", "ACCEPT"}},
{firewalld.IPv4, "filter", "FORWARD", 0, []string{"-s", clusterNetworkCIDR, "-j", "ACCEPT"}},
}
for _, rule := range rules {
err := fw.EnsureRule(rule.ipv, rule.table, rule.chain, rule.priority, rule.args)
if err != nil {
return err
}
}
} else {
dbus := utildbus.New()
ipt := iptables.New(kexec.New(), dbus, iptables.ProtocolIpv4)
_, err := ipt.EnsureRule(iptables.Append, iptables.TableNAT, iptables.ChainPostrouting, "-s", clusterNetworkCIDR, "!", "-d", clusterNetworkCIDR, "-j", "MASQUERADE")
if err != nil {
return err
}
}
return nil
}
func TestEnsureIPTablesMasqRuleAlreadyExists(t *testing.T) {
fcmd := exec.FakeCmd{
CombinedOutputScript: []exec.FakeCombinedOutputAction{
// iptables version check
func() ([]byte, error) { return []byte("iptables v1.9.22"), nil },
// Success.
func() ([]byte, error) { return []byte{}, nil },
},
}
fexec := exec.FakeExec{
CommandScript: []exec.FakeCommandAction{
// iptables version check
func(cmd string, args ...string) exec.Cmd { return exec.InitFakeCmd(&fcmd, cmd, args...) },
// The second Command() call is checking the rule. Success of that exec means "done".
func(cmd string, args ...string) exec.Cmd { return exec.InitFakeCmd(&fcmd, cmd, args...) },
},
}
runner := iptables.New(&fexec, dbus.NewFake(nil, nil), iptables.ProtocolIpv4)
defer runner.Destroy()
err := ensureIPTablesMasqRule(runner, "127.0.0.0/8")
if err != nil {
t.Errorf("expected success, got %v", err)
}
if fcmd.CombinedOutputCalls != 2 {
t.Errorf("expected 2 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls)
}
if !sets.NewString(fcmd.CombinedOutputLog[1]...).HasAll("iptables", "-t", "nat", "-C", "POSTROUTING",
"-m", "comment", "--comment", "kubelet: SNAT outbound cluster traffic",
"!", "-d", "127.0.0.0/8", "-j", "MASQUERADE") {
t.Errorf("wrong CombinedOutput() log, got %#v", fcmd.CombinedOutputLog[1])
}
}
func TestEnsureIPTablesMasqRuleErrorChecking(t *testing.T) {
fcmd := exec.FakeCmd{
CombinedOutputScript: []exec.FakeCombinedOutputAction{
// iptables version check
func() ([]byte, error) { return []byte("iptables v1.9.22"), nil },
// Status 2 on the first call.
func() ([]byte, error) { return nil, &exec.FakeExitError{Status: 2} },
},
}
fexec := exec.FakeExec{
CommandScript: []exec.FakeCommandAction{
// iptables version check
func(cmd string, args ...string) exec.Cmd { return exec.InitFakeCmd(&fcmd, cmd, args...) },
// The second Command() call is checking the rule. Failure of that means create it.
func(cmd string, args ...string) exec.Cmd { return exec.InitFakeCmd(&fcmd, cmd, args...) },
},
}
runner := iptables.New(&fexec, dbus.NewFake(nil, nil), iptables.ProtocolIpv4)
defer runner.Destroy()
err := ensureIPTablesMasqRule(runner, "127.0.0.0/8")
if err == nil {
t.Errorf("expected failure")
}
if fcmd.CombinedOutputCalls != 2 {
t.Errorf("expected 2 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls)
}
}
func newNodeIPTables(clusterNetworkCIDR string, syncPeriod time.Duration) *NodeIPTables {
return &NodeIPTables{
ipt: iptables.New(kexec.New(), utildbus.New(), iptables.ProtocolIpv4),
clusterNetworkCIDR: clusterNetworkCIDR,
syncPeriod: syncPeriod,
}
}
func (oc *OvsController) StartNode(mtu uint) error {
// Assume we are working with IPv4
clusterNetworkCIDR, err := oc.Registry.GetClusterNetworkCIDR()
if err != nil {
log.Errorf("Failed to obtain ClusterNetwork: %v", err)
return err
}
ipt := iptables.New(kexec.New(), utildbus.New(), iptables.ProtocolIpv4)
if err := SetupIptables(ipt, clusterNetworkCIDR); err != nil {
return fmt.Errorf("Failed to set up iptables: %v", err)
}
ipt.AddReloadFunc(func() {
err := SetupIptables(ipt, clusterNetworkCIDR)
if err != nil {
log.Errorf("Error reloading iptables: %v\n", err)
}
})
if err := oc.pluginHooks.PluginStartNode(mtu); err != nil {
return fmt.Errorf("Failed to start plugin: %v", err)
}
oc.markPodNetworkReady()
return nil
}
// RunProxy starts the proxy
func (c *NodeConfig) RunProxy() {
// initialize kube proxy
serviceConfig := pconfig.NewServiceConfig()
endpointsConfig := pconfig.NewEndpointsConfig()
loadBalancer := proxy.NewLoadBalancerRR()
endpointsConfig.RegisterHandler(loadBalancer)
host, _, err := net.SplitHostPort(c.BindAddress)
if err != nil {
glog.Fatalf("The provided value to bind to must be an ip:port %q", c.BindAddress)
}
ip := net.ParseIP(host)
if ip == nil {
glog.Fatalf("The provided value to bind to must be an ip:port: %q", c.BindAddress)
}
protocol := iptables.ProtocolIpv4
if ip.To4() == nil {
protocol = iptables.ProtocolIpv6
}
syncPeriod, err := time.ParseDuration(c.IPTablesSyncPeriod)
if err != nil {
glog.Fatalf("Cannot parse the provided ip-tables sync period (%s) : %v", c.IPTablesSyncPeriod, err)
}
go util.Forever(func() {
proxier, err := proxy.NewProxier(loadBalancer, ip, iptables.New(kexec.New(), protocol), util.PortRange{}, syncPeriod)
if err != nil {
switch {
// conflicting use of iptables, retry
case proxy.IsProxyLocked(err):
glog.Errorf("Unable to start proxy, will retry: %v", err)
return
// on a system without iptables
case strings.Contains(err.Error(), "executable file not found in path"):
glog.V(4).Infof("kube-proxy initialization error: %v", err)
glog.Warningf("WARNING: Could not find the iptables command. The service proxy requires iptables and will be disabled.")
case err == proxy.ErrProxyOnLocalhost:
glog.Warningf("WARNING: The service proxy cannot bind to localhost and will be disabled.")
case strings.Contains(err.Error(), "you must be root"):
glog.Warningf("WARNING: Could not modify iptables. You must run this process as root to use the service proxy.")
default:
glog.Warningf("WARNING: Could not modify iptables. You must run this process as root to use the service proxy: %v", err)
}
select {}
}
pconfig.NewSourceAPI(
c.Client,
30*time.Second,
serviceConfig.Channel("api"),
endpointsConfig.Channel("api"))
serviceConfig.RegisterHandler(proxier)
glog.Infof("Started Kubernetes Proxy on %s", host)
select {}
}, 5*time.Second)
}
// RunProxy starts the proxy
func (c *NodeConfig) RunProxy() {
// initialize kube proxy
serviceConfig := pconfig.NewServiceConfig()
endpointsConfig := pconfig.NewEndpointsConfig()
host, _, err := net.SplitHostPort(c.BindAddress)
if err != nil {
glog.Fatalf("The provided value to bind to must be an ip:port %q", c.BindAddress)
}
ip := net.ParseIP(host)
if ip == nil {
glog.Fatalf("The provided value to bind to must be an ip:port: %q", c.BindAddress)
}
protocol := iptables.ProtocolIpv4
if ip.To4() == nil {
protocol = iptables.ProtocolIpv6
}
syncPeriod, err := time.ParseDuration(c.IPTablesSyncPeriod)
if err != nil {
glog.Fatalf("Cannot parse the provided ip-tables sync period (%s) : %v", c.IPTablesSyncPeriod, err)
}
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartRecordingToSink(c.Client.Events(""))
recorder := eventBroadcaster.NewRecorder(kapi.EventSource{Component: "kube-proxy", Host: c.KubeletConfig.NodeName})
nodeRef := &kapi.ObjectReference{
Kind: "Node",
Name: c.KubeletConfig.NodeName,
}
exec := kexec.New()
dbus := utildbus.New()
iptables := iptables.New(exec, dbus, protocol)
proxier, err := proxy.NewProxier(iptables, exec, syncPeriod, false)
if err != nil {
// This should be fatal, but that would break the integration tests
glog.Warningf("WARNING: Could not initialize Kubernetes Proxy. You must run this process as root to use the service proxy: %v", err)
return
}
iptables.AddReloadFunc(proxier.Sync)
pconfig.NewSourceAPI(
c.Client,
10*time.Minute,
serviceConfig.Channel("api"),
endpointsConfig.Channel("api"))
serviceConfig.RegisterHandler(proxier)
if c.FilteringEndpointsHandler == nil {
endpointsConfig.RegisterHandler(proxier)
} else {
c.FilteringEndpointsHandler.SetBaseEndpointsHandler(proxier)
endpointsConfig.RegisterHandler(c.FilteringEndpointsHandler)
}
recorder.Eventf(nodeRef, kapi.EventTypeNormal, "Starting", "Starting kube-proxy.")
glog.Infof("Started Kubernetes Proxy on %s", host)
}
func NewHostportHandler() HostportHandler {
iptInterface := utiliptables.New(utilexec.New(), utildbus.New(), utiliptables.ProtocolIpv4)
return &handler{
hostPortMap: make(map[hostport]closeable),
iptables: iptInterface,
portOpener: openLocalPort,
}
}
// newIPVSController creates a new controller from the given config.
func newIPVSController(kubeClient *unversioned.Client, namespace string, useUnicast bool, configMapName string) *ipvsControllerController {
ipvsc := ipvsControllerController{
client: kubeClient,
reloadRateLimiter: util.NewTokenBucketRateLimiter(reloadQPS, int(reloadQPS)),
ruCfg: []vip{},
configMapName: configMapName,
}
clusterNodes := getClusterNodesIP(kubeClient)
nodeInfo, err := getNodeInfo(clusterNodes)
if err != nil {
glog.Fatalf("Error getting local IP from nodes in the cluster: %v", err)
}
neighbors := getNodeNeighbors(nodeInfo, clusterNodes)
execer := exec.New()
dbus := utildbus.New()
iptInterface := utiliptables.New(execer, dbus, utiliptables.ProtocolIpv4)
ipvsc.keepalived = &keepalived{
iface: nodeInfo.iface,
ip: nodeInfo.ip,
netmask: nodeInfo.netmask,
nodes: clusterNodes,
neighbors: neighbors,
priority: getNodePriority(nodeInfo.ip, clusterNodes),
useUnicast: useUnicast,
ipt: iptInterface,
}
err = ipvsc.keepalived.loadTemplate()
if err != nil {
glog.Fatalf("Error loading keepalived template: %v", err)
}
eventHandlers := framework.ResourceEventHandlerFuncs{}
ipvsc.svcLister.Store, ipvsc.svcController = framework.NewInformer(
cache.NewListWatchFromClient(
ipvsc.client, "services", namespace, fields.Everything()),
&api.Service{}, resyncPeriod, eventHandlers)
ipvsc.epLister.Store, ipvsc.epController = framework.NewInformer(
cache.NewListWatchFromClient(
ipvsc.client, "endpoints", namespace, fields.Everything()),
&api.Endpoints{}, resyncPeriod, eventHandlers)
return &ipvsc
}
func NewPlugin() network.NetworkPlugin {
protocol := utiliptables.ProtocolIpv4
execer := utilexec.New()
dbus := utildbus.New()
iptInterface := utiliptables.New(execer, dbus, protocol)
return &kubenetNetworkPlugin{
podIPs: make(map[kubecontainer.ContainerID]string),
hostPortMap: make(map[hostport]closeable),
MTU: 1460, //TODO: don't hardcode this
execer: utilexec.New(),
iptables: iptInterface,
}
}
func NewPlugin(networkPluginDir string) network.NetworkPlugin {
protocol := utiliptables.ProtocolIpv4
execer := utilexec.New()
dbus := utildbus.New()
iptInterface := utiliptables.New(execer, dbus, protocol)
return &kubenetNetworkPlugin{
podIPs: make(map[kubecontainer.ContainerID]string),
MTU: 1460, //TODO: don't hardcode this
execer: utilexec.New(),
iptables: iptInterface,
vendorDir: networkPluginDir,
hostportHandler: hostport.NewHostportHandler(),
nonMasqueradeCIDR: "10.0.0.0/8",
}
}
func (oc *OvsController) StartNode(mtu uint) error {
err := oc.initSelfSubnet()
if err != nil {
log.Errorf("Failed to get subnet for this host: %v", err)
return err
}
// Assume we are working with IPv4
clusterNetworkCIDR, err := oc.subnetRegistry.GetClusterNetworkCIDR()
if err != nil {
log.Errorf("Failed to obtain ClusterNetwork: %v", err)
return err
}
servicesNetworkCIDR, err := oc.subnetRegistry.GetServicesNetworkCIDR()
if err != nil {
log.Errorf("Failed to obtain ServicesNetwork: %v", err)
return err
}
err = oc.flowController.Setup(oc.localSubnet.SubnetCIDR, clusterNetworkCIDR, servicesNetworkCIDR, mtu)
if err != nil {
return err
}
ipt := iptables.New(kexec.New(), utildbus.New(), iptables.ProtocolIpv4)
err = SetupIptables(ipt, clusterNetworkCIDR)
if err != nil {
return err
}
ipt.AddReloadFunc(func() {
err := SetupIptables(ipt, clusterNetworkCIDR)
if err != nil {
log.Errorf("Error reloading iptables: %v\n", err)
}
})
result, err := oc.watchAndGetResource("HostSubnet")
if err != nil {
return err
}
subnets := result.([]api.Subnet)
for _, s := range subnets {
oc.flowController.AddOFRules(s.NodeIP, s.SubnetCIDR, oc.localIP)
}
if oc.isMultitenant() {
result, err := oc.watchAndGetResource("NetNamespace")
if err != nil {
return err
}
nslist := result.([]api.NetNamespace)
for _, ns := range nslist {
oc.VNIDMap[ns.Name] = ns.NetID
}
result, err = oc.watchAndGetResource("Service")
if err != nil {
return err
}
services := result.([]api.Service)
for _, svc := range services {
netid, found := oc.VNIDMap[svc.Namespace]
if !found {
return fmt.Errorf("Error fetching Net ID for namespace: %s", svc.Namespace)
}
oc.flowController.AddServiceOFRules(netid, svc.IP, svc.Protocol, svc.Port)
}
_, err = oc.watchAndGetResource("Pod")
if err != nil {
return err
}
}
if oc.ready != nil {
close(oc.ready)
}
return nil
}
// RunProxy starts the proxy
func (c *NodeConfig) RunProxy() {
protocol := utiliptables.ProtocolIpv4
bindAddr := net.ParseIP(c.ProxyConfig.BindAddress)
if bindAddr.To4() == nil {
protocol = utiliptables.ProtocolIpv6
}
portRange := utilnet.ParsePortRangeOrDie(c.ProxyConfig.PortRange)
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartRecordingToSink(c.Client.Events(""))
recorder := eventBroadcaster.NewRecorder(kapi.EventSource{Component: "kube-proxy", Host: c.KubeletConfig.NodeName})
exec := kexec.New()
dbus := utildbus.New()
iptInterface := utiliptables.New(exec, dbus, protocol)
var proxier proxy.ProxyProvider
var endpointsHandler pconfig.EndpointsConfigHandler
switch c.ProxyConfig.Mode {
case "iptables":
glog.V(0).Info("Using iptables Proxier.")
proxierIptables, err := iptables.NewProxier(iptInterface, exec, c.ProxyConfig.IPTablesSyncPeriod.Duration, c.ProxyConfig.MasqueradeAll, *c.ProxyConfig.IPTablesMasqueradeBit)
if err != nil {
if c.Containerized {
glog.Fatalf("error: Could not initialize Kubernetes Proxy: %v\n When running in a container, you must run the container in the host network namespace with --net=host and with --privileged", err)
} else {
glog.Fatalf("error: Could not initialize Kubernetes Proxy. You must run this process as root to use the service proxy: %v", err)
}
}
proxier = proxierIptables
endpointsHandler = proxierIptables
// No turning back. Remove artifacts that might still exist from the userspace Proxier.
glog.V(0).Info("Tearing down userspace rules. Errors here are acceptable.")
userspace.CleanupLeftovers(iptInterface)
case "userspace":
glog.V(0).Info("Using userspace Proxier.")
loadBalancer := userspace.NewLoadBalancerRR()
endpointsHandler = loadBalancer
proxierUserspace, err := userspace.NewProxier(loadBalancer, bindAddr, iptInterface, *portRange, c.ProxyConfig.IPTablesSyncPeriod.Duration, c.ProxyConfig.UDPIdleTimeout.Duration)
if err != nil {
if c.Containerized {
glog.Fatalf("error: Could not initialize Kubernetes Proxy: %v\n When running in a container, you must run the container in the host network namespace with --net=host and with --privileged", err)
} else {
glog.Fatalf("error: Could not initialize Kubernetes Proxy. You must run this process as root to use the service proxy: %v", err)
}
}
proxier = proxierUserspace
// Remove artifacts from the pure-iptables Proxier.
glog.V(0).Info("Tearing down pure-iptables proxy rules. Errors here are acceptable.")
iptables.CleanupLeftovers(iptInterface)
default:
glog.Fatalf("Unknown proxy mode %q", c.ProxyConfig.Mode)
}
iptInterface.AddReloadFunc(proxier.Sync)
// Create configs (i.e. Watches for Services and Endpoints)
// Note: RegisterHandler() calls need to happen before creation of Sources because sources
// only notify on changes, and the initial update (on process start) may be lost if no handlers
// are registered yet.
serviceConfig := pconfig.NewServiceConfig()
serviceConfig.RegisterHandler(proxier)
endpointsConfig := pconfig.NewEndpointsConfig()
if c.FilteringEndpointsHandler == nil {
endpointsConfig.RegisterHandler(endpointsHandler)
} else {
c.FilteringEndpointsHandler.SetBaseEndpointsHandler(endpointsHandler)
endpointsConfig.RegisterHandler(c.FilteringEndpointsHandler)
}
pconfig.NewSourceAPI(
c.Client,
c.ProxyConfig.ConfigSyncPeriod,
serviceConfig.Channel("api"),
endpointsConfig.Channel("api"))
recorder.Eventf(c.ProxyConfig.NodeRef, kapi.EventTypeNormal, "Starting", "Starting kube-proxy.")
glog.Infof("Started Kubernetes Proxy on %s", c.ProxyConfig.BindAddress)
}
// newIPVSController creates a new controller from the given config.
func newIPVSController(kubeClient *unversioned.Client, namespace string, useUnicast bool, configMapName string) *ipvsControllerController {
ipvsc := ipvsControllerController{
client: kubeClient,
reloadRateLimiter: flowcontrol.NewTokenBucketRateLimiter(reloadQPS, int(reloadQPS)),
ruCfg: []vip{},
configMapName: configMapName,
stopCh: make(chan struct{}),
}
podInfo, err := getPodDetails(kubeClient)
if err != nil {
glog.Fatalf("Error getting POD information: %v", err)
}
pod, err := kubeClient.Pods(podInfo.PodNamespace).Get(podInfo.PodName)
if err != nil {
glog.Fatalf("Error getting %v: %v", podInfo.PodName, err)
}
selector := parseNodeSelector(pod.Spec.NodeSelector)
clusterNodes := getClusterNodesIP(kubeClient, selector)
nodeInfo, err := getNetworkInfo(podInfo.NodeIP)
if err != nil {
glog.Fatalf("Error getting local IP from nodes in the cluster: %v", err)
}
neighbors := getNodeNeighbors(nodeInfo, clusterNodes)
execer := exec.New()
dbus := utildbus.New()
iptInterface := utiliptables.New(execer, dbus, utiliptables.ProtocolIpv4)
ipvsc.keepalived = &keepalived{
iface: nodeInfo.iface,
ip: nodeInfo.ip,
netmask: nodeInfo.netmask,
nodes: clusterNodes,
neighbors: neighbors,
priority: getNodePriority(nodeInfo.ip, clusterNodes),
useUnicast: useUnicast,
ipt: iptInterface,
}
ipvsc.syncQueue = NewTaskQueue(ipvsc.sync)
err = ipvsc.keepalived.loadTemplate()
if err != nil {
glog.Fatalf("Error loading keepalived template: %v", err)
}
eventHandlers := cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
ipvsc.syncQueue.enqueue(obj)
},
DeleteFunc: func(obj interface{}) {
ipvsc.syncQueue.enqueue(obj)
},
UpdateFunc: func(old, cur interface{}) {
if !reflect.DeepEqual(old, cur) {
ipvsc.syncQueue.enqueue(cur)
}
},
}
ipvsc.svcLister.Indexer, ipvsc.svcController = cache.NewIndexerInformer(
cache.NewListWatchFromClient(
ipvsc.client, "services", namespace, fields.Everything()),
&api.Service{},
resyncPeriod,
eventHandlers,
cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
ipvsc.epLister.Store, ipvsc.epController = cache.NewInformer(
cache.NewListWatchFromClient(
ipvsc.client, "endpoints", namespace, fields.Everything()),
&api.Endpoints{}, resyncPeriod, eventHandlers)
return &ipvsc
}
// NewProxyServerDefault creates a new ProxyServer object with default parameters.
func NewProxyServerDefault(config *options.ProxyServerConfig) (*ProxyServer, error) {
if c, err := configz.New("componentconfig"); err == nil {
c.Set(config.KubeProxyConfiguration)
} else {
glog.Errorf("unable to register configz: %s", err)
}
protocol := utiliptables.ProtocolIpv4
if net.ParseIP(config.BindAddress).To4() == nil {
protocol = utiliptables.ProtocolIpv6
}
// Create a iptables utils.
execer := exec.New()
dbus := utildbus.New()
iptInterface := utiliptables.New(execer, dbus, protocol)
// We omit creation of pretty much everything if we run in cleanup mode
if config.CleanupAndExit {
return &ProxyServer{
Config: config,
IptInterface: iptInterface,
}, nil
}
// TODO(vmarmol): Use container config for this.
var oomAdjuster *oom.OOMAdjuster
if config.OOMScoreAdj != nil {
oomAdjuster = oom.NewOOMAdjuster()
if err := oomAdjuster.ApplyOOMScoreAdj(0, int(*config.OOMScoreAdj)); err != nil {
glog.V(2).Info(err)
}
}
if config.ResourceContainer != "" {
// Run in its own container.
if err := resourcecontainer.RunInResourceContainer(config.ResourceContainer); err != nil {
glog.Warningf("Failed to start in resource-only container %q: %v", config.ResourceContainer, err)
} else {
glog.V(2).Infof("Running in resource-only container %q", config.ResourceContainer)
}
}
// Create a Kube Client
// define api config source
if config.Kubeconfig == "" && config.Master == "" {
glog.Warningf("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.")
}
// This creates a client, first loading any specified kubeconfig
// file, and then overriding the Master flag, if non-empty.
kubeconfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
&clientcmd.ClientConfigLoadingRules{ExplicitPath: config.Kubeconfig},
&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: config.Master}}).ClientConfig()
if err != nil {
return nil, err
}
kubeconfig.ContentType = config.ContentType
// Override kubeconfig qps/burst settings from flags
kubeconfig.QPS = config.KubeAPIQPS
kubeconfig.Burst = int(config.KubeAPIBurst)
client, err := clientset.NewForConfig(kubeconfig)
if err != nil {
glog.Fatalf("Invalid API configuration: %v", err)
}
// Create event recorder
hostname := nodeutil.GetHostname(config.HostnameOverride)
eventBroadcaster := record.NewBroadcaster()
recorder := eventBroadcaster.NewRecorder(api.EventSource{Component: "kube-proxy", Host: hostname})
var proxier proxy.ProxyProvider
var endpointsHandler proxyconfig.EndpointsConfigHandler
proxyMode := getProxyMode(string(config.Mode), client.Core().Nodes(), hostname, iptInterface, iptables.LinuxKernelCompatTester{})
if proxyMode == proxyModeIPTables {
glog.V(0).Info("Using iptables Proxier.")
if config.IPTablesMasqueradeBit == nil {
// IPTablesMasqueradeBit must be specified or defaulted.
return nil, fmt.Errorf("Unable to read IPTablesMasqueradeBit from config")
}
proxierIPTables, err := iptables.NewProxier(iptInterface, utilsysctl.New(), execer, config.IPTablesSyncPeriod.Duration, config.MasqueradeAll, int(*config.IPTablesMasqueradeBit), config.ClusterCIDR, hostname, getNodeIP(client, hostname))
if err != nil {
glog.Fatalf("Unable to create proxier: %v", err)
}
proxier = proxierIPTables
endpointsHandler = proxierIPTables
// No turning back. Remove artifacts that might still exist from the userspace Proxier.
glog.V(0).Info("Tearing down userspace rules.")
userspace.CleanupLeftovers(iptInterface)
} else {
glog.V(0).Info("Using userspace Proxier.")
// This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for
// our config.EndpointsConfigHandler.
loadBalancer := userspace.NewLoadBalancerRR()
// set EndpointsConfigHandler to our loadBalancer
endpointsHandler = loadBalancer
proxierUserspace, err := userspace.NewProxier(
loadBalancer,
net.ParseIP(config.BindAddress),
iptInterface,
*utilnet.ParsePortRangeOrDie(config.PortRange),
config.IPTablesSyncPeriod.Duration,
config.UDPIdleTimeout.Duration,
)
if err != nil {
glog.Fatalf("Unable to create proxier: %v", err)
}
proxier = proxierUserspace
// Remove artifacts from the pure-iptables Proxier.
glog.V(0).Info("Tearing down pure-iptables proxy rules.")
iptables.CleanupLeftovers(iptInterface)
}
iptInterface.AddReloadFunc(proxier.Sync)
// Create configs (i.e. Watches for Services and Endpoints)
// Note: RegisterHandler() calls need to happen before creation of Sources because sources
// only notify on changes, and the initial update (on process start) may be lost if no handlers
// are registered yet.
serviceConfig := proxyconfig.NewServiceConfig()
serviceConfig.RegisterHandler(proxier)
endpointsConfig := proxyconfig.NewEndpointsConfig()
endpointsConfig.RegisterHandler(endpointsHandler)
proxyconfig.NewSourceAPI(
client.Core().RESTClient(),
config.ConfigSyncPeriod,
serviceConfig.Channel("api"),
endpointsConfig.Channel("api"),
)
config.NodeRef = &api.ObjectReference{
Kind: "Node",
Name: hostname,
UID: types.UID(hostname),
Namespace: "",
}
conntracker := realConntracker{}
return NewProxyServer(client, config, iptInterface, proxier, eventBroadcaster, recorder, conntracker, proxyMode)
}
// RunProxy starts the proxy
func (c *NodeConfig) RunProxy() {
protocol := utiliptables.ProtocolIpv4
bindAddr := net.ParseIP(c.ProxyConfig.BindAddress)
if bindAddr.To4() == nil {
protocol = utiliptables.ProtocolIpv6
}
portRange := utilnet.ParsePortRangeOrDie(c.ProxyConfig.PortRange)
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartRecordingToSink(c.Client.Events(""))
recorder := eventBroadcaster.NewRecorder(kapi.EventSource{Component: "kube-proxy", Host: c.KubeletConfig.NodeName})
execer := kexec.New()
dbus := utildbus.New()
iptInterface := utiliptables.New(execer, dbus, protocol)
var proxier proxy.ProxyProvider
var endpointsHandler pconfig.EndpointsConfigHandler
switch c.ProxyConfig.Mode {
case componentconfig.ProxyModeIPTables:
glog.V(0).Info("Using iptables Proxier.")
if c.ProxyConfig.IPTablesMasqueradeBit == nil {
// IPTablesMasqueradeBit must be specified or defaulted.
glog.Fatalf("Unable to read IPTablesMasqueradeBit from config")
}
proxierIptables, err := iptables.NewProxier(iptInterface, execer, c.ProxyConfig.IPTablesSyncPeriod.Duration, c.ProxyConfig.MasqueradeAll, int(*c.ProxyConfig.IPTablesMasqueradeBit), c.ProxyConfig.ClusterCIDR)
if err != nil {
if c.Containerized {
glog.Fatalf("error: Could not initialize Kubernetes Proxy: %v\n When running in a container, you must run the container in the host network namespace with --net=host and with --privileged", err)
} else {
glog.Fatalf("error: Could not initialize Kubernetes Proxy. You must run this process as root to use the service proxy: %v", err)
}
}
proxier = proxierIptables
endpointsHandler = proxierIptables
// No turning back. Remove artifacts that might still exist from the userspace Proxier.
glog.V(0).Info("Tearing down userspace rules.")
userspace.CleanupLeftovers(iptInterface)
case componentconfig.ProxyModeUserspace:
glog.V(0).Info("Using userspace Proxier.")
// This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for
// our config.EndpointsConfigHandler.
loadBalancer := userspace.NewLoadBalancerRR()
// set EndpointsConfigHandler to our loadBalancer
endpointsHandler = loadBalancer
proxierUserspace, err := userspace.NewProxier(
loadBalancer,
bindAddr,
iptInterface,
*portRange,
c.ProxyConfig.IPTablesSyncPeriod.Duration,
c.ProxyConfig.UDPIdleTimeout.Duration,
)
if err != nil {
if c.Containerized {
glog.Fatalf("error: Could not initialize Kubernetes Proxy: %v\n When running in a container, you must run the container in the host network namespace with --net=host and with --privileged", err)
} else {
glog.Fatalf("error: Could not initialize Kubernetes Proxy. You must run this process as root to use the service proxy: %v", err)
}
}
proxier = proxierUserspace
// Remove artifacts from the pure-iptables Proxier.
glog.V(0).Info("Tearing down pure-iptables proxy rules.")
iptables.CleanupLeftovers(iptInterface)
default:
glog.Fatalf("Unknown proxy mode %q", c.ProxyConfig.Mode)
}
// Create configs (i.e. Watches for Services and Endpoints)
// Note: RegisterHandler() calls need to happen before creation of Sources because sources
// only notify on changes, and the initial update (on process start) may be lost if no handlers
// are registered yet.
serviceConfig := pconfig.NewServiceConfig()
if c.EnableUnidling {
unidlingLoadBalancer := ouserspace.NewLoadBalancerRR()
signaler := unidler.NewEventSignaler(recorder)
unidlingUserspaceProxy, err := unidler.NewUnidlerProxier(unidlingLoadBalancer, bindAddr, iptInterface, execer, *portRange, c.ProxyConfig.IPTablesSyncPeriod.Duration, c.ProxyConfig.UDPIdleTimeout.Duration, signaler)
if err != nil {
if c.Containerized {
glog.Fatalf("error: Could not initialize Kubernetes Proxy: %v\n When running in a container, you must run the container in the host network namespace with --net=host and with --privileged", err)
} else {
glog.Fatalf("error: Could not initialize Kubernetes Proxy. You must run this process as root to use the service proxy: %v", err)
}
}
hybridProxier, err := hybrid.NewHybridProxier(unidlingLoadBalancer, unidlingUserspaceProxy, endpointsHandler, proxier, c.ProxyConfig.IPTablesSyncPeriod.Duration, serviceConfig)
if err != nil {
if c.Containerized {
glog.Fatalf("error: Could not initialize Kubernetes Proxy: %v\n When running in a container, you must run the container in the host network namespace with --net=host and with --privileged", err)
} else {
glog.Fatalf("error: Could not initialize Kubernetes Proxy. You must run this process as root to use the service proxy: %v", err)
}
}
endpointsHandler = hybridProxier
iptInterface.AddReloadFunc(hybridProxier.Sync)
serviceConfig.RegisterHandler(hybridProxier)
}
endpointsConfig := pconfig.NewEndpointsConfig()
// customized handling registration that inserts a filter if needed
if c.FilteringEndpointsHandler != nil {
if err := c.FilteringEndpointsHandler.Start(endpointsHandler); err != nil {
glog.Fatalf("error: node proxy plugin startup failed: %v", err)
}
endpointsHandler = c.FilteringEndpointsHandler
}
endpointsConfig.RegisterHandler(endpointsHandler)
c.ServiceStore = pconfig.NewServiceStore(c.ServiceStore, serviceConfig.Channel("api"))
c.EndpointsStore = pconfig.NewEndpointsStore(c.EndpointsStore, endpointsConfig.Channel("api"))
// will be started by RunServiceStores
recorder.Eventf(c.ProxyConfig.NodeRef, kapi.EventTypeNormal, "Starting", "Starting kube-proxy.")
// periodically sync k8s iptables rules
go utilwait.Forever(proxier.SyncLoop, 0)
glog.Infof("Started Kubernetes Proxy on %s", c.ProxyConfig.BindAddress)
}
// Run runs the specified ProxyServer. This should never exit (unless CleanupAndExit is set).
func (s *ProxyServer) Run(_ []string) error {
protocol := utiliptables.ProtocolIpv4
if s.BindAddress.To4() == nil {
protocol = utiliptables.ProtocolIpv6
}
// remove iptables rules and exit
if s.CleanupAndExit {
execer := exec.New()
ipt := utiliptables.New(execer, protocol)
encounteredError := userspace.CleanupLeftovers(ipt)
encounteredError = iptables.CleanupLeftovers(ipt) || encounteredError
if encounteredError {
return errors.New("Encountered an error while tearing down rules.")
}
return nil
}
// TODO(vmarmol): Use container config for this.
oomAdjuster := oom.NewOomAdjuster()
if err := oomAdjuster.ApplyOomScoreAdj(0, s.OOMScoreAdj); err != nil {
glog.V(2).Info(err)
}
// Run in its own container.
if err := util.RunInResourceContainer(s.ResourceContainer); err != nil {
glog.Warningf("Failed to start in resource-only container %q: %v", s.ResourceContainer, err)
} else {
glog.V(2).Infof("Running in resource-only container %q", s.ResourceContainer)
}
// define api config source
if s.Kubeconfig == "" && s.Master == "" {
glog.Warningf("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.")
}
// This creates a client, first loading any specified kubeconfig
// file, and then overriding the Master flag, if non-empty.
kubeconfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
&clientcmd.ClientConfigLoadingRules{ExplicitPath: s.Kubeconfig},
&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: s.Master}}).ClientConfig()
if err != nil {
return err
}
client, err := client.New(kubeconfig)
if err != nil {
glog.Fatalf("Invalid API configuration: %v", err)
}
// Add event recorder
Hostname := nodeutil.GetHostname(s.HostnameOverride)
eventBroadcaster := record.NewBroadcaster()
s.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "kube-proxy", Host: Hostname})
eventBroadcaster.StartRecordingToSink(client.Events(""))
s.nodeRef = &api.ObjectReference{
Kind: "Node",
Name: Hostname,
UID: types.UID(Hostname),
Namespace: "",
}
// Birth Cry
s.birthCry()
serviceConfig := config.NewServiceConfig()
endpointsConfig := config.NewEndpointsConfig()
var proxier proxy.ProxyProvider
var endpointsHandler config.EndpointsConfigHandler
// guaranteed false on error, error only necessary for debugging
shouldUseIptables, err := iptables.ShouldUseIptablesProxier()
if err != nil {
glog.Errorf("Can't determine whether to use iptables or userspace, using userspace proxier: %v", err)
}
if !s.ForceUserspaceProxy && shouldUseIptables {
glog.V(2).Info("Using iptables Proxier.")
execer := exec.New()
ipt := utiliptables.New(execer, protocol)
proxierIptables, err := iptables.NewProxier(ipt, execer, s.SyncPeriod, s.MasqueradeAll)
if err != nil {
glog.Fatalf("Unable to create proxier: %v", err)
}
proxier = proxierIptables
endpointsHandler = proxierIptables
// No turning back. Remove artifacts that might still exist from the userspace Proxier.
glog.V(2).Info("Tearing down userspace rules. Errors here are acceptable.")
userspace.CleanupLeftovers(ipt)
} else {
glog.V(2).Info("Using userspace Proxier.")
// This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for
// our config.EndpointsConfigHandler.
loadBalancer := userspace.NewLoadBalancerRR()
// set EndpointsConfigHandler to our loadBalancer
endpointsHandler = loadBalancer
execer := exec.New()
ipt := utiliptables.New(execer, protocol)
proxierUserspace, err := userspace.NewProxier(loadBalancer, s.BindAddress, ipt, s.PortRange, s.SyncPeriod)
if err != nil {
glog.Fatalf("Unable to create proxer: %v", err)
}
proxier = proxierUserspace
// Remove artifacts from the pure-iptables Proxier.
glog.V(2).Info("Tearing down pure-iptables proxy rules. Errors here are acceptable.")
iptables.CleanupLeftovers(ipt)
}
// Wire proxier to handle changes to services
serviceConfig.RegisterHandler(proxier)
// And wire endpointsHandler to handle changes to endpoints to services
endpointsConfig.RegisterHandler(endpointsHandler)
// Note: RegisterHandler() calls need to happen before creation of Sources because sources
// only notify on changes, and the initial update (on process start) may be lost if no handlers
// are registered yet.
config.NewSourceAPI(
client,
30*time.Second,
serviceConfig.Channel("api"),
endpointsConfig.Channel("api"),
)
if s.HealthzPort > 0 {
go util.Until(func() {
err := http.ListenAndServe(s.HealthzBindAddress.String()+":"+strconv.Itoa(s.HealthzPort), nil)
if err != nil {
glog.Errorf("Starting health server failed: %v", err)
}
}, 5*time.Second, util.NeverStop)
}
// Just loop forever for now...
proxier.SyncLoop()
return nil
}
// RunProxy starts the proxy
func (c *NodeConfig) RunProxy(endpointsFilterer FilteringEndpointsConfigHandler) {
// initialize kube proxy
serviceConfig := pconfig.NewServiceConfig()
endpointsConfig := pconfig.NewEndpointsConfig()
loadBalancer := proxy.NewLoadBalancerRR()
if endpointsFilterer == nil {
endpointsConfig.RegisterHandler(loadBalancer)
} else {
endpointsFilterer.SetBaseEndpointsHandler(loadBalancer)
endpointsConfig.RegisterHandler(endpointsFilterer)
}
host, _, err := net.SplitHostPort(c.BindAddress)
if err != nil {
glog.Fatalf("The provided value to bind to must be an ip:port %q", c.BindAddress)
}
ip := net.ParseIP(host)
if ip == nil {
glog.Fatalf("The provided value to bind to must be an ip:port: %q", c.BindAddress)
}
protocol := iptables.ProtocolIpv4
if ip.To4() == nil {
protocol = iptables.ProtocolIpv6
}
syncPeriod, err := time.ParseDuration(c.IPTablesSyncPeriod)
if err != nil {
glog.Fatalf("Cannot parse the provided ip-tables sync period (%s) : %v", c.IPTablesSyncPeriod, err)
}
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartRecordingToSink(c.Client.Events(""))
recorder := eventBroadcaster.NewRecorder(kapi.EventSource{Component: "kube-proxy", Host: c.KubeletConfig.NodeName})
nodeRef := &kapi.ObjectReference{
Kind: "Node",
Name: c.KubeletConfig.NodeName,
}
go util.Forever(func() {
dbus := utildbus.New()
iptables := iptables.New(kexec.New(), dbus, protocol)
proxier, err := proxy.NewProxier(loadBalancer, ip, iptables, util.PortRange{}, syncPeriod)
iptables.AddReloadFunc(proxier.Sync)
if err != nil {
switch {
// conflicting use of iptables, retry
case proxy.IsProxyLocked(err):
glog.Errorf("Unable to start proxy, will retry: %v", err)
return
// on a system without iptables
case strings.Contains(err.Error(), "executable file not found in path"):
glog.V(4).Infof("kube-proxy initialization error: %v", err)
glog.Warningf("WARNING: Could not find the iptables command. The service proxy requires iptables and will be disabled.")
case err == proxy.ErrProxyOnLocalhost:
glog.Warningf("WARNING: The service proxy cannot bind to localhost and will be disabled.")
case strings.Contains(err.Error(), "you must be root"):
glog.Warningf("WARNING: Could not modify iptables. You must run this process as root to use the service proxy.")
default:
glog.Warningf("WARNING: Could not modify iptables. You must run this process as root to use the service proxy: %v", err)
}
select {}
}
pconfig.NewSourceAPI(
c.Client,
10*time.Minute,
serviceConfig.Channel("api"),
endpointsConfig.Channel("api"))
serviceConfig.RegisterHandler(proxier)
recorder.Eventf(nodeRef, "Starting", "Starting kube-proxy.")
glog.Infof("Started Kubernetes Proxy on %s", host)
select {}
}, 5*time.Second)
}
// NewFlannelHelper creates a new flannel helper.
func NewFlannelHelper() *FlannelHelper {
return &FlannelHelper{
subnetFile: flannelSubnetFile,
iptablesHelper: utiliptables.New(utilexec.New(), utildbus.New(), utiliptables.ProtocolIpv4),
}
}
// Run runs the specified ProxyServer. This should never exit.
func (s *ProxyServer) Run(_ []string) error {
// TODO(vmarmol): Use container config for this.
oomAdjuster := oom.NewOomAdjuster()
if err := oomAdjuster.ApplyOomScoreAdj(0, s.OOMScoreAdj); err != nil {
glog.V(2).Info(err)
}
// Run in its own container.
if err := util.RunInResourceContainer(s.ResourceContainer); err != nil {
glog.Warningf("Failed to start in resource-only container %q: %v", s.ResourceContainer, err)
} else {
glog.V(2).Infof("Running in resource-only container %q", s.ResourceContainer)
}
serviceConfig := config.NewServiceConfig()
endpointsConfig := config.NewEndpointsConfig()
protocol := iptables.ProtocolIpv4
if s.BindAddress.To4() == nil {
protocol = iptables.ProtocolIpv6
}
loadBalancer := userspace.NewLoadBalancerRR()
proxier, err := userspace.NewProxier(loadBalancer, s.BindAddress, iptables.New(exec.New(), protocol), s.PortRange)
if err != nil {
glog.Fatalf("Unable to create proxer: %v", err)
}
// Wire proxier to handle changes to services
serviceConfig.RegisterHandler(proxier)
// And wire loadBalancer to handle changes to endpoints to services
endpointsConfig.RegisterHandler(loadBalancer)
// Note: RegisterHandler() calls need to happen before creation of Sources because sources
// only notify on changes, and the initial update (on process start) may be lost if no handlers
// are registered yet.
// define api config source
if s.Kubeconfig == "" && s.Master == "" {
glog.Warningf("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.")
}
// This creates a client, first loading any specified kubeconfig
// file, and then overriding the Master flag, if non-empty.
kubeconfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
&clientcmd.ClientConfigLoadingRules{ExplicitPath: s.Kubeconfig},
&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: s.Master}}).ClientConfig()
if err != nil {
return err
}
client, err := client.New(kubeconfig)
if err != nil {
glog.Fatalf("Invalid API configuration: %v", err)
}
config.NewSourceAPI(
client,
30*time.Second,
serviceConfig.Channel("api"),
endpointsConfig.Channel("api"),
)
if s.HealthzPort > 0 {
go util.Forever(func() {
err := http.ListenAndServe(s.HealthzBindAddress.String()+":"+strconv.Itoa(s.HealthzPort), nil)
if err != nil {
glog.Errorf("Starting health server failed: %v", err)
}
}, 5*time.Second)
}
// Just loop forever for now...
proxier.SyncLoop()
return nil
}
func (plugin *OsdnNode) SetupSDN() (bool, error) {
clusterNetworkCIDR := plugin.networkInfo.ClusterNetwork.String()
serviceNetworkCIDR := plugin.networkInfo.ServiceNetwork.String()
localSubnetCIDR := plugin.localSubnetCIDR
_, ipnet, err := net.ParseCIDR(localSubnetCIDR)
localSubnetMaskLength, _ := ipnet.Mask.Size()
localSubnetGateway := netutils.GenerateDefaultGateway(ipnet).String()
glog.V(5).Infof("[SDN setup] node pod subnet %s gateway %s", ipnet.String(), localSubnetGateway)
exec := kexec.New()
if plugin.clearLbr0IptablesRule {
// Delete docker's left-over lbr0 rule; cannot do this from
// NewNodePlugin (where docker is cleaned up) because we need
// localSubnetCIDR which is only valid after plugin start
ipt := iptables.New(exec, utildbus.New(), iptables.ProtocolIpv4)
ipt.DeleteRule(iptables.TableNAT, iptables.ChainPostrouting, "-s", localSubnetCIDR, "!", "-o", "lbr0", "-j", "MASQUERADE")
}
gwCIDR := fmt.Sprintf("%s/%d", localSubnetGateway, localSubnetMaskLength)
if plugin.alreadySetUp(gwCIDR, clusterNetworkCIDR) {
glog.V(5).Infof("[SDN setup] no SDN setup required")
return false, nil
}
glog.V(5).Infof("[SDN setup] full SDN setup required")
if err := os.MkdirAll("/run/openshift-sdn", 0700); err != nil {
return false, err
}
config := fmt.Sprintf("export OPENSHIFT_CLUSTER_SUBNET=%s", clusterNetworkCIDR)
err = ioutil.WriteFile("/run/openshift-sdn/config.env", []byte(config), 0644)
if err != nil {
return false, err
}
err = plugin.ovs.AddBridge("fail-mode=secure", "protocols=OpenFlow13")
if err != nil {
return false, err
}
_ = plugin.ovs.DeletePort(VXLAN)
_, err = plugin.ovs.AddPort(VXLAN, 1, "type=vxlan", `options:remote_ip="flow"`, `options:key="flow"`)
if err != nil {
return false, err
}
_ = plugin.ovs.DeletePort(TUN)
_, err = plugin.ovs.AddPort(TUN, 2, "type=internal")
if err != nil {
return false, err
}
otx := plugin.ovs.NewTransaction()
// Table 0: initial dispatch based on in_port
// vxlan0
otx.AddFlow("table=0, priority=200, in_port=1, arp, nw_src=%s, nw_dst=%s, actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:1", clusterNetworkCIDR, localSubnetCIDR)
otx.AddFlow("table=0, priority=200, in_port=1, ip, nw_src=%s, nw_dst=%s, actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:1", clusterNetworkCIDR, localSubnetCIDR)
otx.AddFlow("table=0, priority=150, in_port=1, actions=drop")
// tun0
otx.AddFlow("table=0, priority=200, in_port=2, arp, nw_src=%s, nw_dst=%s, actions=goto_table:5", localSubnetGateway, clusterNetworkCIDR)
otx.AddFlow("table=0, priority=200, in_port=2, ip, actions=goto_table:5")
otx.AddFlow("table=0, priority=150, in_port=2, actions=drop")
// else, from a container
otx.AddFlow("table=0, priority=100, arp, actions=goto_table:2")
otx.AddFlow("table=0, priority=100, ip, actions=goto_table:2")
otx.AddFlow("table=0, priority=0, actions=drop")
// Table 1: VXLAN ingress filtering; filled in by AddHostSubnetRules()
// eg, "table=1, priority=100, tun_src=${remote_node_ip}, actions=goto_table:5"
otx.AddFlow("table=1, priority=0, actions=drop")
// Table 2: from OpenShift container; validate IP/MAC, assign tenant-id; filled in by openshift-sdn-ovs
// eg, "table=2, priority=100, in_port=${ovs_port}, arp, nw_src=${ipaddr}, arp_sha=${macaddr}, actions=load:${tenant_id}->NXM_NX_REG0[], goto_table:5"
// "table=2, priority=100, in_port=${ovs_port}, ip, nw_src=${ipaddr}, actions=load:${tenant_id}->NXM_NX_REG0[], goto_table:3"
// (${tenant_id} is always 0 for single-tenant)
otx.AddFlow("table=2, priority=0, actions=drop")
// Table 3: from OpenShift container; service vs non-service
otx.AddFlow("table=3, priority=100, ip, nw_dst=%s, actions=goto_table:4", serviceNetworkCIDR)
otx.AddFlow("table=3, priority=0, actions=goto_table:5")
// Table 4: from OpenShift container; service dispatch; filled in by AddServiceRules()
otx.AddFlow("table=4, priority=200, reg0=0, actions=output:2")
// eg, "table=4, priority=100, reg0=${tenant_id}, ${service_proto}, nw_dst=${service_ip}, tp_dst=${service_port}, actions=output:2"
otx.AddFlow("table=4, priority=0, actions=drop")
// Table 5: general routing
otx.AddFlow("table=5, priority=300, arp, nw_dst=%s, actions=output:2", localSubnetGateway)
otx.AddFlow("table=5, priority=300, ip, nw_dst=%s, actions=output:2", localSubnetGateway)
otx.AddFlow("table=5, priority=200, arp, nw_dst=%s, actions=goto_table:6", localSubnetCIDR)
otx.AddFlow("table=5, priority=200, ip, nw_dst=%s, actions=goto_table:7", localSubnetCIDR)
otx.AddFlow("table=5, priority=100, arp, nw_dst=%s, actions=goto_table:8", clusterNetworkCIDR)
otx.AddFlow("table=5, priority=100, ip, nw_dst=%s, actions=goto_table:8", clusterNetworkCIDR)
otx.AddFlow("table=5, priority=0, ip, actions=goto_table:9")
otx.AddFlow("table=5, priority=0, arp, actions=drop")
// Table 6: ARP to container, filled in by openshift-sdn-ovs
// eg, "table=6, priority=100, arp, nw_dst=${container_ip}, actions=output:${ovs_port}"
otx.AddFlow("table=6, priority=0, actions=drop")
// Table 7: IP to container; filled in by openshift-sdn-ovs
// eg, "table=7, priority=100, reg0=0, ip, nw_dst=${ipaddr}, actions=output:${ovs_port}"
// eg, "table=7, priority=100, reg0=${tenant_id}, ip, nw_dst=${ipaddr}, actions=output:${ovs_port}"
otx.AddFlow("table=7, priority=0, actions=drop")
// Table 8: to remote container; filled in by AddHostSubnetRules()
// eg, "table=8, priority=100, arp, nw_dst=${remote_subnet_cidr}, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31], set_field:${remote_node_ip}->tun_dst,output:1"
// eg, "table=8, priority=100, ip, nw_dst=${remote_subnet_cidr}, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31], set_field:${remote_node_ip}->tun_dst,output:1"
otx.AddFlow("table=8, priority=0, actions=drop")
// Table 9: egress network policy dispatch; edited by updateEgressNetworkPolicyRules()
// eg, "table=9, reg0=${tenant_id}, priority=2, ip, nw_dst=${external_cidr}, actions=drop
otx.AddFlow("table=9, priority=0, actions=output:2")
err = otx.EndTransaction()
if err != nil {
return false, err
}
itx := ipcmd.NewTransaction(exec, TUN)
itx.AddAddress(gwCIDR)
defer deleteLocalSubnetRoute(TUN, localSubnetCIDR)
itx.SetLink("mtu", fmt.Sprint(plugin.mtu))
itx.SetLink("up")
itx.AddRoute(clusterNetworkCIDR, "proto", "kernel", "scope", "link")
itx.AddRoute(serviceNetworkCIDR)
err = itx.EndTransaction()
if err != nil {
return false, err
}
sysctl := sysctl.New()
// Enable IP forwarding for ipv4 packets
err = sysctl.SetSysctl("net/ipv4/ip_forward", 1)
if err != nil {
return false, fmt.Errorf("Could not enable IPv4 forwarding: %s", err)
}
err = sysctl.SetSysctl(fmt.Sprintf("net/ipv4/conf/%s/forwarding", TUN), 1)
if err != nil {
return false, fmt.Errorf("Could not enable IPv4 forwarding on %s: %s", TUN, err)
}
// Table 253: rule version; note action is hex bytes separated by '.'
otx = plugin.ovs.NewTransaction()
pluginVersion := getPluginVersion(plugin.multitenant)
otx.AddFlow("%s, %s%s.%s", VERSION_TABLE, VERSION_ACTION, pluginVersion[0], pluginVersion[1])
err = otx.EndTransaction()
if err != nil {
return false, err
}
return true, nil
}
// NewProxyServerDefault creates a new ProxyServer object with default parameters.
func NewProxyServerDefault(config *ProxyServerConfig) (*ProxyServer, error) {
protocol := utiliptables.ProtocolIpv4
if config.BindAddress.To4() == nil {
protocol = utiliptables.ProtocolIpv6
}
// Create a iptables utils.
execer := exec.New()
dbus := utildbus.New()
iptInterface := utiliptables.New(execer, dbus, protocol)
// We ommit creation of pretty much everything if we run in cleanup mode
if config.CleanupAndExit {
return &ProxyServer{
Config: config,
IptInterface: iptInterface,
}, nil
}
// TODO(vmarmol): Use container config for this.
var oomAdjuster *oom.OOMAdjuster
if config.OOMScoreAdj != 0 {
oomAdjuster = oom.NewOOMAdjuster()
if err := oomAdjuster.ApplyOOMScoreAdj(0, config.OOMScoreAdj); err != nil {
glog.V(2).Info(err)
}
}
if config.ResourceContainer != "" {
// Run in its own container.
if err := util.RunInResourceContainer(config.ResourceContainer); err != nil {
glog.Warningf("Failed to start in resource-only container %q: %v", config.ResourceContainer, err)
} else {
glog.V(2).Infof("Running in resource-only container %q", config.ResourceContainer)
}
}
// Create a Kube Client
// define api config source
if config.Kubeconfig == "" && config.Master == "" {
glog.Warningf("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.")
}
// This creates a client, first loading any specified kubeconfig
// file, and then overriding the Master flag, if non-empty.
kubeconfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
&clientcmd.ClientConfigLoadingRules{ExplicitPath: config.Kubeconfig},
&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: config.Master}}).ClientConfig()
if err != nil {
return nil, err
}
// Override kubeconfig qps/burst settings from flags
kubeconfig.QPS = config.KubeApiQps
kubeconfig.Burst = config.KubeApiBurst
client, err := kubeclient.New(kubeconfig)
if err != nil {
glog.Fatalf("Invalid API configuration: %v", err)
}
// Create event recorder
hostname := nodeutil.GetHostname(config.HostnameOverride)
eventBroadcaster := record.NewBroadcaster()
recorder := eventBroadcaster.NewRecorder(api.EventSource{Component: "kube-proxy", Host: hostname})
eventBroadcaster.StartRecordingToSink(client.Events(""))
var proxier proxy.ProxyProvider
var endpointsHandler proxyconfig.EndpointsConfigHandler
useIptablesProxy := false
if mayTryIptablesProxy(config.ProxyMode, client.Nodes(), hostname) {
var err error
// guaranteed false on error, error only necessary for debugging
useIptablesProxy, err = iptables.ShouldUseIptablesProxier()
if err != nil {
glog.Errorf("Can't determine whether to use iptables proxy, using userspace proxier: %v", err)
}
}
if useIptablesProxy {
glog.V(2).Info("Using iptables Proxier.")
proxierIptables, err := iptables.NewProxier(iptInterface, execer, config.IptablesSyncPeriod, config.MasqueradeAll)
if err != nil {
glog.Fatalf("Unable to create proxier: %v", err)
}
proxier = proxierIptables
endpointsHandler = proxierIptables
// No turning back. Remove artifacts that might still exist from the userspace Proxier.
glog.V(2).Info("Tearing down userspace rules. Errors here are acceptable.")
userspace.CleanupLeftovers(iptInterface)
} else {
glog.V(2).Info("Using userspace Proxier.")
// This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for
// our config.EndpointsConfigHandler.
loadBalancer := userspace.NewLoadBalancerRR()
// set EndpointsConfigHandler to our loadBalancer
endpointsHandler = loadBalancer
proxierUserspace, err := userspace.NewProxier(loadBalancer, config.BindAddress, iptInterface, config.PortRange, config.IptablesSyncPeriod)
if err != nil {
glog.Fatalf("Unable to create proxier: %v", err)
}
proxier = proxierUserspace
// Remove artifacts from the pure-iptables Proxier.
glog.V(2).Info("Tearing down pure-iptables proxy rules. Errors here are acceptable.")
iptables.CleanupLeftovers(iptInterface)
}
iptInterface.AddReloadFunc(proxier.Sync)
// Create configs (i.e. Watches for Services and Endpoints)
// Note: RegisterHandler() calls need to happen before creation of Sources because sources
// only notify on changes, and the initial update (on process start) may be lost if no handlers
// are registered yet.
serviceConfig := proxyconfig.NewServiceConfig()
serviceConfig.RegisterHandler(proxier)
endpointsConfig := proxyconfig.NewEndpointsConfig()
endpointsConfig.RegisterHandler(endpointsHandler)
proxyconfig.NewSourceAPI(
client,
config.ConfigSyncPeriod,
serviceConfig.Channel("api"),
endpointsConfig.Channel("api"),
)
config.nodeRef = &api.ObjectReference{
Kind: "Node",
Name: hostname,
UID: types.UID(hostname),
Namespace: "",
}
return NewProxyServer(config, iptInterface, proxier, recorder)
}