// Control Inter Network Communication. Install/remove only if it is not/is present.
func setINC(network1, network2 string, enable bool) error {
var (
table = iptables.Filter
chain = "FORWARD"
args = [2][]string{{"-s", network1, "-d", network2, "-j", "DROP"}, {"-s", network2, "-d", network1, "-j", "DROP"}}
)
if enable {
for i := 0; i < 2; i++ {
if iptables.Exists(table, chain, args[i]...) {
continue
}
if output, err := iptables.Raw(append([]string{"-I", chain}, args[i]...)...); err != nil {
return fmt.Errorf("unable to add inter-network communication rule: %s", err.Error())
} else if len(output) != 0 {
return fmt.Errorf("error adding inter-network communication rule: %s", string(output))
}
}
} else {
for i := 0; i < 2; i++ {
if !iptables.Exists(table, chain, args[i]...) {
continue
}
if output, err := iptables.Raw(append([]string{"-D", chain}, args[i]...)...); err != nil {
return fmt.Errorf("unable to remove inter-network communication rule: %s", err.Error())
} else if len(output) != 0 {
return fmt.Errorf("error removing inter-network communication rule: %s", string(output))
}
}
}
return nil
}
// adapted from https://github.com/docker/libnetwork/blob/master/iptables/iptables.go
func (p *portMapper) forward(action iptables.Action, ip net.IP, port int, proto, destAddr string, destPort int, srcIface, destIface string) error {
daddr := ip.String()
if ip == nil || ip.IsUnspecified() {
// iptables interprets "0.0.0.0" as "0.0.0.0/32", whereas we
// want "0.0.0.0/0". "0/0" is correctly interpreted as "any
// value" by both iptables and ip6tables.
daddr = "0/0"
}
args := []string{"-t", string(iptables.Nat), string(action), "VIC",
"-i", srcIface,
"-p", proto,
"-d", daddr,
"--dport", strconv.Itoa(port),
"-j", "DNAT",
"--to-destination", net.JoinHostPort(destAddr, strconv.Itoa(destPort))}
if output, err := iptables.Raw(args...); err != nil {
return err
} else if len(output) != 0 {
return iptables.ChainError{Chain: "FORWARD", Output: output}
}
ipStr := ""
if ip != nil && !ip.IsUnspecified() {
ipStr = ip.String()
}
switch action {
case iptables.Append:
p.bindings[bindKey{ipStr, port}] = nil
case iptables.Delete:
delete(p.bindings, bindKey{ipStr, port})
}
if output, err := iptables.Raw("-t", string(iptables.Filter), string(action), "VIC",
"-i", srcIface,
"-o", destIface,
"-p", proto,
"-d", destAddr,
"--dport", strconv.Itoa(destPort),
"-j", "ACCEPT"); err != nil {
return err
} else if len(output) != 0 {
return iptables.ChainError{Chain: "FORWARD", Output: output}
}
if output, err := iptables.Raw("-t", string(iptables.Nat), string(action), "POSTROUTING",
"-p", proto,
"-d", destAddr,
"--dport", strconv.Itoa(destPort),
"-j", "MASQUERADE"); err != nil {
return err
} else if len(output) != 0 {
return iptables.ChainError{Chain: "FORWARD", Output: output}
}
return nil
}
func programChainRule(rule iptRule, ruleDescr string, insert bool) error {
var (
prefix []string
operation string
condition bool
doesExist = iptables.Exists(rule.table, rule.chain, rule.args...)
)
if insert {
condition = !doesExist
prefix = []string{"-I", rule.chain}
operation = "enable"
} else {
condition = doesExist
prefix = []string{"-D", rule.chain}
operation = "disable"
}
if rule.preArgs != nil {
prefix = append(rule.preArgs, prefix...)
}
if condition {
if output, err := iptables.Raw(append(prefix, rule.args...)...); err != nil {
return fmt.Errorf("Unable to %s %s rule: %s", operation, ruleDescr, err.Error())
} else if len(output) != 0 {
return &iptables.ChainError{Chain: rule.chain, Output: output}
}
}
return nil
}
func chainExists(cname string) bool {
if _, err := iptables.Raw("-L", cname); err != nil {
return false
}
return true
}
func verifyV4INCEntries(networks map[string]*bridgeNetwork, numEntries int, t *testing.T) {
out, err := iptables.Raw("-nvL", IsolationChain)
if err != nil {
t.Fatal(err)
}
found := 0
for _, x := range networks {
for _, y := range networks {
if x == y {
continue
}
re := regexp.MustCompile(fmt.Sprintf("%s %s", x.config.BridgeName, y.config.BridgeName))
matches := re.FindAllString(string(out[:]), -1)
if len(matches) != 1 {
t.Fatalf("Cannot find expected inter-network isolation rules in IP Tables:\n%s", string(out[:]))
}
found++
}
}
if found != numEntries {
t.Fatalf("Cannot find expected number (%d) of inter-network isolation rules in IP Tables:\n%s\nFound %d", numEntries, string(out[:]), found)
}
}
// iptablesRunAndCheck runs an iptables command with the provided args
func iptablesRunAndCheck(action iptables.Action, args []string) error {
args = append([]string{string(action)}, args...)
if output, err := iptables.Raw(args...); err != nil {
return err
} else if len(output) != 0 {
return iptables.ChainError{Chain: "FORWARD", Output: output}
}
return nil
}
func rawIPTables(args ...string) error {
if output, err := iptables.Raw(args...); err != nil {
return fmt.Errorf("unable to add overlay filter: %v", err)
} else if len(output) != 0 {
return fmt.Errorf("unable to add overlay filter: %s", string(output))
}
return nil
}
func setIcc(bridgeIface string, iccEnable, insert bool) error {
var (
table = iptables.Filter
chain = "FORWARD"
args = []string{"-i", bridgeIface, "-o", bridgeIface, "-j"}
acceptArgs = append(args, "ACCEPT")
dropArgs = append(args, "DROP")
)
if insert {
if !iccEnable {
iptables.Raw(append([]string{"-D", chain}, acceptArgs...)...)
if !iptables.Exists(table, chain, dropArgs...) {
if err := iptables.RawCombinedOutput(append([]string{"-A", chain}, dropArgs...)...); err != nil {
return fmt.Errorf("Unable to prevent intercontainer communication: %s", err.Error())
}
}
} else {
iptables.Raw(append([]string{"-D", chain}, dropArgs...)...)
if !iptables.Exists(table, chain, acceptArgs...) {
if err := iptables.RawCombinedOutput(append([]string{"-I", chain}, acceptArgs...)...); err != nil {
return fmt.Errorf("Unable to allow intercontainer communication: %s", err.Error())
}
}
}
} else {
// Remove any ICC rule.
if !iccEnable {
if iptables.Exists(table, chain, dropArgs...) {
iptables.Raw(append([]string{"-D", chain}, dropArgs...)...)
}
} else {
if iptables.Exists(table, chain, acceptArgs...) {
iptables.Raw(append([]string{"-D", chain}, acceptArgs...)...)
}
}
}
return nil
}
func (driver *driver) natOut() error {
masquerade := []string{
"POSTROUTING", "-t", "nat",
"-s", driver.pluginConfig.containerSubnet.String(),
"-j", "MASQUERADE",
}
if _, err := iptables.Raw(
append([]string{"-C"}, masquerade...)...,
); err != nil {
incl := append([]string{"-I"}, masquerade...)
if output, err := iptables.Raw(incl...); err != nil {
return err
} else if len(output) > 0 {
return &iptables.ChainError{
Chain: "POSTROUTING",
Output: output,
}
}
}
return nil
}
// todo: reconcile with what libnetwork does and port mappings
func natOut(cidr string) error {
masquerade := []string{
"POSTROUTING", "-t", "nat",
"-s", cidr,
"-j", "MASQUERADE",
}
if _, err := iptables.Raw(
append([]string{"-C"}, masquerade...)...,
); err != nil {
incl := append([]string{"-I"}, masquerade...)
if output, err := iptables.Raw(incl...); err != nil {
return err
} else if len(output) > 0 {
return &iptables.ChainError{
Chain: "POSTROUTING",
Output: output,
}
}
}
return nil
}
func verifyV4INCEntries(networks map[string]*bridgeNetwork, numEntries int, t *testing.T) {
out, err := iptables.Raw("-L", "FORWARD")
if err != nil {
t.Fatal(err)
}
for _, nw := range networks {
nt := types.GetIPNetCopy(nw.bridge.bridgeIPv4)
nt.IP = nt.IP.Mask(nt.Mask)
re := regexp.MustCompile(nt.String())
matches := re.FindAllString(string(out[:]), -1)
if len(matches) != numEntries {
t.Fatalf("Cannot find expected inter-network isolation rules in IP Tables:\n%s", string(out[:]))
}
}
}
func TestLinkContainers(t *testing.T) {
defer netutils.SetupTestNetNS(t)()
d := newDriver()
config := &configuration{
EnableIPTables: true,
}
genericOption := make(map[string]interface{})
genericOption[netlabel.GenericData] = config
if err := d.Config(genericOption); err != nil {
t.Fatalf("Failed to setup driver config: %v", err)
}
netconfig := &networkConfiguration{
BridgeName: DefaultBridgeName,
EnableICC: false,
}
genericOption = make(map[string]interface{})
genericOption[netlabel.GenericData] = netconfig
err := d.CreateNetwork("net1", genericOption)
if err != nil {
t.Fatalf("Failed to create bridge: %v", err)
}
exposedPorts := getExposedPorts()
epOptions := make(map[string]interface{})
epOptions[netlabel.ExposedPorts] = exposedPorts
te1 := &testEndpoint{ifaces: []*testInterface{}}
err = d.CreateEndpoint("net1", "ep1", te1, epOptions)
if err != nil {
t.Fatalf("Failed to create an endpoint : %s", err.Error())
}
addr1 := te1.ifaces[0].addr
if addr1.IP.To4() == nil {
t.Fatalf("No Ipv4 address assigned to the endpoint: ep1")
}
te2 := &testEndpoint{ifaces: []*testInterface{}}
err = d.CreateEndpoint("net1", "ep2", te2, nil)
if err != nil {
t.Fatalf("Failed to create an endpoint : %s", err.Error())
}
addr2 := te2.ifaces[0].addr
if addr2.IP.To4() == nil {
t.Fatalf("No Ipv4 address assigned to the endpoint: ep2")
}
ce := []string{"ep1"}
cConfig := &containerConfiguration{ChildEndpoints: ce}
genericOption = make(map[string]interface{})
genericOption[netlabel.GenericData] = cConfig
err = d.Join("net1", "ep2", "", te2, genericOption)
if err != nil {
t.Fatalf("Failed to link ep1 and ep2")
}
out, err := iptables.Raw("-L", DockerChain)
for _, pm := range exposedPorts {
regex := fmt.Sprintf("%s dpt:%d", pm.Proto.String(), pm.Port)
re := regexp.MustCompile(regex)
matches := re.FindAllString(string(out[:]), -1)
if len(matches) != 1 {
t.Fatalf("IP Tables programming failed %s", string(out[:]))
}
regex = fmt.Sprintf("%s spt:%d", pm.Proto.String(), pm.Port)
matched, _ := regexp.MatchString(regex, string(out[:]))
if !matched {
t.Fatalf("IP Tables programming failed %s", string(out[:]))
}
}
err = d.Leave("net1", "ep2")
if err != nil {
t.Fatalf("Failed to unlink ep1 and ep2")
}
out, err = iptables.Raw("-L", DockerChain)
for _, pm := range exposedPorts {
regex := fmt.Sprintf("%s dpt:%d", pm.Proto.String(), pm.Port)
re := regexp.MustCompile(regex)
matches := re.FindAllString(string(out[:]), -1)
if len(matches) != 0 {
t.Fatalf("Leave should have deleted relevant IPTables rules %s", string(out[:]))
}
regex = fmt.Sprintf("%s spt:%d", pm.Proto.String(), pm.Port)
matched, _ := regexp.MatchString(regex, string(out[:]))
if matched {
t.Fatalf("Leave should have deleted relevant IPTables rules %s", string(out[:]))
}
}
// Error condition test with an invalid endpoint-id "ep4"
ce = []string{"ep1", "ep4"}
cConfig = &containerConfiguration{ChildEndpoints: ce}
genericOption = make(map[string]interface{})
genericOption[netlabel.GenericData] = cConfig
err = d.Join("net1", "ep2", "", te2, genericOption)
if err != nil {
out, err = iptables.Raw("-L", DockerChain)
for _, pm := range exposedPorts {
regex := fmt.Sprintf("%s dpt:%d", pm.Proto.String(), pm.Port)
re := regexp.MustCompile(regex)
matches := re.FindAllString(string(out[:]), -1)
if len(matches) != 0 {
t.Fatalf("Error handling should rollback relevant IPTables rules %s", string(out[:]))
}
regex = fmt.Sprintf("%s spt:%d", pm.Proto.String(), pm.Port)
matched, _ := regexp.MatchString(regex, string(out[:]))
if matched {
t.Fatalf("Error handling should rollback relevant IPTables rules %s", string(out[:]))
}
}
} else {
t.Fatalf("Expected Join to fail given link conditions are not satisfied")
}
}
func TestLinkContainers(t *testing.T) {
if !testutils.IsRunningInContainer() {
defer testutils.SetupTestOSContext(t)()
}
d := newDriver()
config := &configuration{
EnableIPTables: true,
}
genericOption := make(map[string]interface{})
genericOption[netlabel.GenericData] = config
if err := d.configure(genericOption); err != nil {
t.Fatalf("Failed to setup driver config: %v", err)
}
netconfig := &networkConfiguration{
BridgeName: DefaultBridgeName,
EnableICC: false,
}
genericOption = make(map[string]interface{})
genericOption[netlabel.GenericData] = netconfig
ipdList := getIPv4Data(t, "")
err := d.CreateNetwork("net1", genericOption, nil, ipdList, nil)
if err != nil {
t.Fatalf("Failed to create bridge: %v", err)
}
te1 := newTestEndpoint(ipdList[0].Pool, 11)
err = d.CreateEndpoint("net1", "ep1", te1.Interface(), nil)
if err != nil {
t.Fatalf("Failed to create an endpoint : %s", err.Error())
}
exposedPorts := getExposedPorts()
sbOptions := make(map[string]interface{})
sbOptions[netlabel.ExposedPorts] = exposedPorts
err = d.Join("net1", "ep1", "sbox", te1, sbOptions)
if err != nil {
t.Fatalf("Failed to join the endpoint: %v", err)
}
err = d.ProgramExternalConnectivity("net1", "ep1", sbOptions)
if err != nil {
t.Fatalf("Failed to program external connectivity: %v", err)
}
addr1 := te1.iface.addr
if addr1.IP.To4() == nil {
t.Fatalf("No Ipv4 address assigned to the endpoint: ep1")
}
te2 := newTestEndpoint(ipdList[0].Pool, 22)
err = d.CreateEndpoint("net1", "ep2", te2.Interface(), nil)
if err != nil {
t.Fatalf("Failed to create an endpoint : %s", err.Error())
}
addr2 := te2.iface.addr
if addr2.IP.To4() == nil {
t.Fatalf("No Ipv4 address assigned to the endpoint: ep2")
}
sbOptions = make(map[string]interface{})
sbOptions[netlabel.GenericData] = options.Generic{
"ChildEndpoints": []string{"ep1"},
}
err = d.Join("net1", "ep2", "", te2, sbOptions)
if err != nil {
t.Fatalf("Failed to link ep1 and ep2")
}
err = d.ProgramExternalConnectivity("net1", "ep2", sbOptions)
if err != nil {
t.Fatalf("Failed to program external connectivity: %v", err)
}
out, err := iptables.Raw("-L", DockerChain)
for _, pm := range exposedPorts {
regex := fmt.Sprintf("%s dpt:%d", pm.Proto.String(), pm.Port)
re := regexp.MustCompile(regex)
matches := re.FindAllString(string(out[:]), -1)
if len(matches) != 1 {
t.Fatalf("IP Tables programming failed %s", string(out[:]))
}
regex = fmt.Sprintf("%s spt:%d", pm.Proto.String(), pm.Port)
matched, _ := regexp.MatchString(regex, string(out[:]))
if !matched {
t.Fatalf("IP Tables programming failed %s", string(out[:]))
}
}
err = d.RevokeExternalConnectivity("net1", "ep2")
if err != nil {
t.Fatalf("Failed to revoke external connectivity: %v", err)
}
err = d.Leave("net1", "ep2")
if err != nil {
t.Fatalf("Failed to unlink ep1 and ep2")
}
out, err = iptables.Raw("-L", DockerChain)
for _, pm := range exposedPorts {
regex := fmt.Sprintf("%s dpt:%d", pm.Proto.String(), pm.Port)
re := regexp.MustCompile(regex)
matches := re.FindAllString(string(out[:]), -1)
if len(matches) != 0 {
t.Fatalf("Leave should have deleted relevant IPTables rules %s", string(out[:]))
}
regex = fmt.Sprintf("%s spt:%d", pm.Proto.String(), pm.Port)
matched, _ := regexp.MatchString(regex, string(out[:]))
if matched {
t.Fatalf("Leave should have deleted relevant IPTables rules %s", string(out[:]))
}
}
// Error condition test with an invalid endpoint-id "ep4"
sbOptions = make(map[string]interface{})
sbOptions[netlabel.GenericData] = options.Generic{
"ChildEndpoints": []string{"ep1", "ep4"},
}
err = d.Join("net1", "ep2", "", te2, sbOptions)
if err != nil {
t.Fatal(err)
}
err = d.ProgramExternalConnectivity("net1", "ep2", sbOptions)
if err != nil {
out, err = iptables.Raw("-L", DockerChain)
for _, pm := range exposedPorts {
regex := fmt.Sprintf("%s dpt:%d", pm.Proto.String(), pm.Port)
re := regexp.MustCompile(regex)
matches := re.FindAllString(string(out[:]), -1)
if len(matches) != 0 {
t.Fatalf("Error handling should rollback relevant IPTables rules %s", string(out[:]))
}
regex = fmt.Sprintf("%s spt:%d", pm.Proto.String(), pm.Port)
matched, _ := regexp.MatchString(regex, string(out[:]))
if matched {
t.Fatalf("Error handling should rollback relevant IPTables rules %s", string(out[:]))
}
}
} else {
t.Fatalf("Expected Join to fail given link conditions are not satisfied")
}
}