func setNetworkChain(cname string, remove bool) error {
// Initialize the onetime global overlay chain
filterOnce.Do(setupGlobalChain)
exists := chainExists(cname)
opt := "-N"
// In case of remove, make sure to flush the rules in the chain
if remove && exists {
if err := iptables.RawCombinedOutput("-F", cname); err != nil {
return fmt.Errorf("failed to flush overlay network chain %s rules: %v", cname, err)
}
opt = "-X"
}
if (!remove && !exists) || (remove && exists) {
if err := iptables.RawCombinedOutput(opt, cname); err != nil {
return fmt.Errorf("failed network chain operation %q for chain %s: %v", opt, cname, err)
}
}
if !remove {
if !iptables.Exists(iptables.Filter, cname, "-j", "DROP") {
if err := iptables.RawCombinedOutput("-A", cname, "-j", "DROP"); err != nil {
return fmt.Errorf("failed adding default drop rule to overlay network chain %s: %v", cname, err)
}
}
}
return nil
}
// Control Inter Network Communication. Install/remove only if it is not/is present.
func setINC(iface1, iface2 string, enable bool) error {
var (
table = iptables.Filter
chain = IsolationChain
args = [2][]string{{"-i", iface1, "-o", iface2, "-j", "DROP"}, {"-i", iface2, "-o", iface1, "-j", "DROP"}}
)
if enable {
for i := 0; i < 2; i++ {
if iptables.Exists(table, chain, args[i]...) {
continue
}
if err := iptables.RawCombinedOutput(append([]string{"-I", chain}, args[i]...)...); err != nil {
return fmt.Errorf("unable to add inter-network communication rule: %v", err)
}
}
} else {
for i := 0; i < 2; i++ {
if !iptables.Exists(table, chain, args[i]...) {
continue
}
if err := iptables.RawCombinedOutput(append([]string{"-D", chain}, args[i]...)...); err != nil {
return fmt.Errorf("unable to remove inter-network communication rule: %v", err)
}
}
}
return nil
}
func setupGlobalChain() {
if err := iptables.RawCombinedOutput("-N", globalChain); err != nil {
logrus.Errorf("could not create global overlay chain: %v", err)
return
}
if err := iptables.RawCombinedOutput("-A", globalChain, "-j", "RETURN"); err != nil {
logrus.Errorf("could not install default return chain in the overlay global chain: %v", err)
return
}
}
// In the filter table FORWARD chain first rule should be to jump to INGRESS-CHAIN
// This chain has the rules to allow access to the published ports for swarm tasks
// from local bridge networks and docker_gwbridge (ie:taks on other swarm netwroks)
func arrangeIngressFilterRule() {
if iptables.ExistChain(ingressChain, iptables.Filter) {
if iptables.Exists(iptables.Filter, "FORWARD", "-j", ingressChain) {
if err := iptables.RawCombinedOutput("-D", "FORWARD", "-j", ingressChain); err != nil {
logrus.Warnf("failed to delete jump rule to ingressChain in filter table: %v", err)
}
}
if err := iptables.RawCombinedOutput("-I", "FORWARD", "-j", ingressChain); err != nil {
logrus.Warnf("failed to add jump rule to ingressChain in filter table: %v", err)
}
}
}
func setupGlobalChain() {
// Because of an ungraceful shutdown, chain could already be present
if !chainExists(globalChain) {
if err := iptables.RawCombinedOutput("-N", globalChain); err != nil {
logrus.Errorf("could not create global overlay chain: %v", err)
return
}
}
if !iptables.Exists(iptables.Filter, globalChain, "-j", "RETURN") {
if err := iptables.RawCombinedOutput("-A", globalChain, "-j", "RETURN"); err != nil {
logrus.Errorf("could not install default return chain in the overlay global chain: %v", err)
}
}
}
func programMangle(vni uint32, add bool) (err error) {
var (
p = strconv.FormatUint(uint64(vxlanPort), 10)
c = fmt.Sprintf("0>>22&0x3C@12&0xFFFFFF00=%d", int(vni)<<8)
m = strconv.FormatUint(uint64(mark), 10)
chain = "OUTPUT"
rule = []string{"-p", "udp", "--dport", p, "-m", "u32", "--u32", c, "-j", "MARK", "--set-mark", m}
a = "-A"
action = "install"
)
if add == iptables.Exists(iptables.Mangle, chain, rule...) {
return
}
if !add {
a = "-D"
action = "remove"
}
if err = iptables.RawCombinedOutput(append([]string{"-t", string(iptables.Mangle), a, chain}, rule...)...); err != nil {
logrus.Warnf("could not %s mangle rule: %v", action, err)
}
return
}
func programChainRule(rule iptRule, ruleDescr string, insert bool) error {
var (
prefix []string
operation string
condition bool
doesExist = iptables.Exists(rule.table, rule.chain, rule.args...)
)
if insert {
condition = !doesExist
prefix = []string{"-I", rule.chain}
operation = "enable"
} else {
condition = doesExist
prefix = []string{"-D", rule.chain}
operation = "disable"
}
if rule.preArgs != nil {
prefix = append(rule.preArgs, prefix...)
}
if condition {
if err := iptables.RawCombinedOutput(append(prefix, rule.args...)...); err != nil {
return fmt.Errorf("Unable to %s %s rule: %s", operation, ruleDescr, err.Error())
}
}
return nil
}
func (r *resolver) SetupFunc() func() {
return (func() {
var err error
addr := &net.UDPAddr{
IP: net.ParseIP(resolverIP),
}
r.conn, err = net.ListenUDP("udp", addr)
if err != nil {
r.err = fmt.Errorf("error in opening name server socket %v", err)
return
}
laddr := r.conn.LocalAddr()
_, ipPort, _ := net.SplitHostPort(laddr.String())
rules := [][]string{
{"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", laddr.String()},
{"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
}
for _, rule := range rules {
r.err = iptables.RawCombinedOutput(rule...)
if r.err != nil {
return
}
}
r.err = nil
})
}
func setIcc(bridgeIface string, iccEnable, insert bool) error {
var (
table = iptables.Filter
chain = "FORWARD"
args = []string{"-i", bridgeIface, "-o", bridgeIface, "-j"}
acceptArgs = append(args, "ACCEPT")
dropArgs = append(args, "DROP")
)
if insert {
if !iccEnable {
iptables.Raw(append([]string{"-D", chain}, acceptArgs...)...)
if !iptables.Exists(table, chain, dropArgs...) {
if err := iptables.RawCombinedOutput(append([]string{"-A", chain}, dropArgs...)...); err != nil {
return fmt.Errorf("Unable to prevent intercontainer communication: %s", err.Error())
}
}
} else {
iptables.Raw(append([]string{"-D", chain}, dropArgs...)...)
if !iptables.Exists(table, chain, acceptArgs...) {
if err := iptables.RawCombinedOutput(append([]string{"-I", chain}, acceptArgs...)...); err != nil {
return fmt.Errorf("Unable to allow intercontainer communication: %s", err.Error())
}
}
}
} else {
// Remove any ICC rule.
if !iccEnable {
if iptables.Exists(table, chain, dropArgs...) {
iptables.Raw(append([]string{"-D", chain}, dropArgs...)...)
}
} else {
if iptables.Exists(table, chain, acceptArgs...) {
iptables.Raw(append([]string{"-D", chain}, acceptArgs...)...)
}
}
}
return nil
}
// Ensure the jump rule is on top
func ensureJumpRule(fromChain, toChain string) error {
var (
table = iptables.Filter
args = []string{"-j", toChain}
)
if iptables.Exists(table, fromChain, args...) {
err := iptables.RawCombinedOutput(append([]string{"-D", fromChain}, args...)...)
if err != nil {
return fmt.Errorf("unable to remove jump to %s rule in %s chain: %s", toChain, fromChain, err.Error())
}
}
err := iptables.RawCombinedOutput(append([]string{"-I", fromChain}, args...)...)
if err != nil {
return fmt.Errorf("unable to insert jump to %s rule in %s chain: %s", toChain, fromChain, err.Error())
}
return nil
}
func setFilters(cname, brName string, remove bool) error {
opt := "-I"
if remove {
opt = "-D"
}
// Every time we set filters for a new subnet make sure to move the global overlay hook to the top of the both the OUTPUT and forward chains
if !remove {
for _, chain := range []string{"OUTPUT", "FORWARD"} {
exists := iptables.Exists(iptables.Filter, chain, "-j", globalChain)
if exists {
if err := iptables.RawCombinedOutput("-D", chain, "-j", globalChain); err != nil {
return fmt.Errorf("failed to delete overlay hook in chain %s while moving the hook: %v", chain, err)
}
}
if err := iptables.RawCombinedOutput("-I", chain, "-j", globalChain); err != nil {
return fmt.Errorf("failed to insert overlay hook in chain %s: %v", chain, err)
}
}
}
// Insert/Delete the rule to jump to per-bridge chain
exists := iptables.Exists(iptables.Filter, globalChain, "-o", brName, "-j", cname)
if (!remove && !exists) || (remove && exists) {
if err := iptables.RawCombinedOutput(opt, globalChain, "-o", brName, "-j", cname); err != nil {
return fmt.Errorf("failed to add per-bridge filter rule for bridge %s, network chain %s: %v", brName, cname, err)
}
}
exists = iptables.Exists(iptables.Filter, cname, "-i", brName, "-j", "ACCEPT")
if (!remove && exists) || (remove && !exists) {
return nil
}
if err := iptables.RawCombinedOutput(opt, cname, "-i", brName, "-j", "ACCEPT"); err != nil {
return fmt.Errorf("failed to add overlay filter rile for network chain %s, bridge %s: %v", cname, brName, err)
}
return nil
}
func addReturnRule(chain string) error {
var (
table = iptables.Filter
args = []string{"-j", "RETURN"}
)
if iptables.Exists(table, chain, args...) {
return nil
}
err := iptables.RawCombinedOutput(append([]string{"-I", chain}, args...)...)
if err != nil {
return fmt.Errorf("unable to add return rule in %s chain: %s", chain, err.Error())
}
return nil
}
func programIngress(gwIP net.IP, ingressPorts []*PortConfig, isDelete bool) error {
addDelOpt := "-I"
if isDelete {
addDelOpt = "-D"
}
chainExists := iptables.ExistChain(ingressChain, iptables.Nat)
ingressOnce.Do(func() {
if chainExists {
// Flush ingress chain rules during init if it
// exists. It might contain stale rules from
// previous life.
if err := iptables.RawCombinedOutput("-t", "nat", "-F", ingressChain); err != nil {
logrus.Errorf("Could not flush ingress chain rules during init: %v", err)
}
}
})
if !isDelete {
if !chainExists {
if err := iptables.RawCombinedOutput("-t", "nat", "-N", ingressChain); err != nil {
return fmt.Errorf("failed to create ingress chain: %v", err)
}
}
if !iptables.Exists(iptables.Nat, ingressChain, "-j", "RETURN") {
if err := iptables.RawCombinedOutput("-t", "nat", "-A", ingressChain, "-j", "RETURN"); err != nil {
return fmt.Errorf("failed to add return rule in ingress chain: %v", err)
}
}
for _, chain := range []string{"OUTPUT", "PREROUTING"} {
if !iptables.Exists(iptables.Nat, chain, "-j", ingressChain) {
if err := iptables.RawCombinedOutput("-t", "nat", "-I", chain, "-j", ingressChain); err != nil {
return fmt.Errorf("failed to add jump rule in %s to ingress chain: %v", chain, err)
}
}
}
oifName, err := findOIFName(gwIP)
if err != nil {
return fmt.Errorf("failed to find gateway bridge interface name for %s: %v", gwIP, err)
}
path := filepath.Join("/proc/sys/net/ipv4/conf", oifName, "route_localnet")
if err := ioutil.WriteFile(path, []byte{'1', '\n'}, 0644); err != nil {
return fmt.Errorf("could not write to %s: %v", path, err)
}
ruleArgs := strings.Fields(fmt.Sprintf("-m addrtype --src-type LOCAL -o %s -j MASQUERADE", oifName))
if !iptables.Exists(iptables.Nat, "POSTROUTING", ruleArgs...) {
if err := iptables.RawCombinedOutput(append([]string{"-t", "nat", "-I", "POSTROUTING"}, ruleArgs...)...); err != nil {
return fmt.Errorf("failed to add ingress localhost POSTROUTING rule for %s: %v", oifName, err)
}
}
}
for _, iPort := range ingressPorts {
if iptables.ExistChain(ingressChain, iptables.Nat) {
rule := strings.Fields(fmt.Sprintf("-t nat %s %s -p %s --dport %d -j DNAT --to-destination %s:%d",
addDelOpt, ingressChain, strings.ToLower(PortConfig_Protocol_name[int32(iPort.Protocol)]), iPort.PublishedPort, gwIP, iPort.PublishedPort))
if err := iptables.RawCombinedOutput(rule...); err != nil {
return fmt.Errorf("setting up rule failed, %v: %v", rule, err)
}
}
if err := plumbProxy(iPort, isDelete); err != nil {
return fmt.Errorf("failed to create proxy for port %d: %v", iPort.PublishedPort, err)
}
}
return nil
}
func programIngress(gwIP net.IP, ingressPorts []*PortConfig, isDelete bool) error {
addDelOpt := "-I"
if isDelete {
addDelOpt = "-D"
}
chainExists := iptables.ExistChain(ingressChain, iptables.Nat)
filterChainExists := iptables.ExistChain(ingressChain, iptables.Filter)
ingressOnce.Do(func() {
// Flush nat table and filter table ingress chain rules during init if it
// exists. It might contain stale rules from previous life.
if chainExists {
if err := iptables.RawCombinedOutput("-t", "nat", "-F", ingressChain); err != nil {
logrus.Errorf("Could not flush nat table ingress chain rules during init: %v", err)
}
}
if filterChainExists {
if err := iptables.RawCombinedOutput("-F", ingressChain); err != nil {
logrus.Errorf("Could not flush filter table ingress chain rules during init: %v", err)
}
}
})
if !isDelete {
if !chainExists {
if err := iptables.RawCombinedOutput("-t", "nat", "-N", ingressChain); err != nil {
return fmt.Errorf("failed to create ingress chain: %v", err)
}
}
if !filterChainExists {
if err := iptables.RawCombinedOutput("-N", ingressChain); err != nil {
return fmt.Errorf("failed to create filter table ingress chain: %v", err)
}
}
if !iptables.Exists(iptables.Nat, ingressChain, "-j", "RETURN") {
if err := iptables.RawCombinedOutput("-t", "nat", "-A", ingressChain, "-j", "RETURN"); err != nil {
return fmt.Errorf("failed to add return rule in nat table ingress chain: %v", err)
}
}
if !iptables.Exists(iptables.Filter, ingressChain, "-j", "RETURN") {
if err := iptables.RawCombinedOutput("-A", ingressChain, "-j", "RETURN"); err != nil {
return fmt.Errorf("failed to add return rule to filter table ingress chain: %v", err)
}
}
for _, chain := range []string{"OUTPUT", "PREROUTING"} {
if !iptables.Exists(iptables.Nat, chain, "-m", "addrtype", "--dst-type", "LOCAL", "-j", ingressChain) {
if err := iptables.RawCombinedOutput("-t", "nat", "-I", chain, "-m", "addrtype", "--dst-type", "LOCAL", "-j", ingressChain); err != nil {
return fmt.Errorf("failed to add jump rule in %s to ingress chain: %v", chain, err)
}
}
}
if !iptables.Exists(iptables.Filter, "FORWARD", "-j", ingressChain) {
if err := iptables.RawCombinedOutput("-I", "FORWARD", "-j", ingressChain); err != nil {
return fmt.Errorf("failed to add jump rule to %s in filter table forward chain: %v", ingressChain, err)
}
}
oifName, err := findOIFName(gwIP)
if err != nil {
return fmt.Errorf("failed to find gateway bridge interface name for %s: %v", gwIP, err)
}
path := filepath.Join("/proc/sys/net/ipv4/conf", oifName, "route_localnet")
if err := ioutil.WriteFile(path, []byte{'1', '\n'}, 0644); err != nil {
return fmt.Errorf("could not write to %s: %v", path, err)
}
ruleArgs := strings.Fields(fmt.Sprintf("-m addrtype --src-type LOCAL -o %s -j MASQUERADE", oifName))
if !iptables.Exists(iptables.Nat, "POSTROUTING", ruleArgs...) {
if err := iptables.RawCombinedOutput(append([]string{"-t", "nat", "-I", "POSTROUTING"}, ruleArgs...)...); err != nil {
return fmt.Errorf("failed to add ingress localhost POSTROUTING rule for %s: %v", oifName, err)
}
}
}
for _, iPort := range ingressPorts {
if iptables.ExistChain(ingressChain, iptables.Nat) {
rule := strings.Fields(fmt.Sprintf("-t nat %s %s -p %s --dport %d -j DNAT --to-destination %s:%d",
addDelOpt, ingressChain, strings.ToLower(PortConfig_Protocol_name[int32(iPort.Protocol)]), iPort.PublishedPort, gwIP, iPort.PublishedPort))
if err := iptables.RawCombinedOutput(rule...); err != nil {
errStr := fmt.Sprintf("setting up rule failed, %v: %v", rule, err)
if !isDelete {
return fmt.Errorf("%s", errStr)
}
logrus.Infof("%s", errStr)
}
}
// Filter table rules to allow a published service to be accessible in the local node from..
// 1) service tasks attached to other networks
// 2) unmanaged containers on bridge networks
rule := strings.Fields(fmt.Sprintf("%s %s -m state -p %s --sport %d --state ESTABLISHED,RELATED -j ACCEPT",
addDelOpt, ingressChain, strings.ToLower(PortConfig_Protocol_name[int32(iPort.Protocol)]), iPort.PublishedPort))
if err := iptables.RawCombinedOutput(rule...); err != nil {
errStr := fmt.Sprintf("setting up rule failed, %v: %v", rule, err)
if !isDelete {
return fmt.Errorf("%s", errStr)
}
logrus.Warnf("%s", errStr)
}
rule = strings.Fields(fmt.Sprintf("%s %s -p %s --dport %d -j ACCEPT",
addDelOpt, ingressChain, strings.ToLower(PortConfig_Protocol_name[int32(iPort.Protocol)]), iPort.PublishedPort))
if err := iptables.RawCombinedOutput(rule...); err != nil {
errStr := fmt.Sprintf("setting up rule failed, %v: %v", rule, err)
if !isDelete {
return fmt.Errorf("%s", errStr)
}
logrus.Warnf("%s", errStr)
}
if err := plumbProxy(iPort, isDelete); err != nil {
logrus.Warnf("failed to create proxy for port %d: %v", iPort.PublishedPort, err)
}
}
return nil
}
