func createBridge(name, network, natIface string) (*Bridge, error) {
ipAddr, ipNet, err := net.ParseCIDR(network)
if err != nil {
return nil, err
}
if err := netlink.CreateBridge(name, true); err != nil {
return nil, err
}
iface, err := net.InterfaceByName(name)
if err != nil {
return nil, err
}
// We need to explicitly assign the MAC address to avoid it changing to a lower value
// See: https://github.com/flynn/flynn/issues/223
b := random.Bytes(5)
mac := fmt.Sprintf("fe:%02x:%02x:%02x:%02x:%02x", b[0], b[1], b[2], b[3], b[4])
if err := netlink.NetworkSetMacAddress(iface, mac); err != nil {
return nil, err
}
if err := netlink.NetworkLinkAddIp(iface, ipAddr, ipNet); err != nil {
return nil, err
}
if err := netlink.NetworkLinkUp(iface); err != nil {
return nil, err
}
if err := ioutil.WriteFile("/proc/sys/net/ipv4/ip_forward", []byte("1\n"), 0644); err != nil {
return nil, err
}
if err := setupIPTables(name, natIface); err != nil {
return nil, err
}
bridge := &Bridge{
name: name,
iface: iface,
ipAddr: ipAddr,
ipNet: ipNet,
alloc: ipallocator.New(),
}
bridge.alloc.RequestIP(ipNet, ipAddr)
return bridge, nil
}
// ConfigureNetworking is called once during host startup and passed the
// strategy and identifier of the networking coordinatior job. Currently the
// only strategy implemented uses flannel.
func (l *LibvirtLXCBackend) ConfigureNetworking(config *host.NetworkConfig) error {
var err error
l.bridgeAddr, l.bridgeNet, err = net.ParseCIDR(config.Subnet)
if err != nil {
return err
}
l.ipalloc.RequestIP(l.bridgeNet, l.bridgeAddr)
err = netlink.CreateBridge(bridgeName, false)
bridgeExists := os.IsExist(err)
if err != nil && !bridgeExists {
return err
}
bridge, err := net.InterfaceByName(bridgeName)
if err != nil {
return err
}
if !bridgeExists {
// We need to explicitly assign the MAC address to avoid it changing to a lower value
// See: https://github.com/flynn/flynn/issues/223
b := random.Bytes(5)
bridgeMAC := fmt.Sprintf("fe:%02x:%02x:%02x:%02x:%02x", b[0], b[1], b[2], b[3], b[4])
if err := netlink.NetworkSetMacAddress(bridge, bridgeMAC); err != nil {
return err
}
}
currAddrs, err := bridge.Addrs()
if err != nil {
return err
}
setIP := true
for _, addr := range currAddrs {
ip, net, _ := net.ParseCIDR(addr.String())
if ip.Equal(l.bridgeAddr) && net.String() == l.bridgeNet.String() {
setIP = false
} else {
if err := netlink.NetworkLinkDelIp(bridge, ip, net); err != nil {
return err
}
}
}
if setIP {
if err := netlink.NetworkLinkAddIp(bridge, l.bridgeAddr, l.bridgeNet); err != nil {
return err
}
}
if err := netlink.NetworkLinkUp(bridge); err != nil {
return err
}
network, err := l.libvirt.LookupNetworkByName(libvirtNetName)
if err != nil {
// network doesn't exist
networkConfig := <.Network{
Name: libvirtNetName,
Bridge: lt.Bridge{Name: bridgeName},
Forward: lt.Forward{Mode: "bridge"},
}
network, err = l.libvirt.NetworkDefineXML(string(networkConfig.XML()))
if err != nil {
return err
}
}
active, err := network.IsActive()
if err != nil {
return err
}
if !active {
if err := network.Create(); err != nil {
return err
}
}
if defaultNet, err := l.libvirt.LookupNetworkByName("default"); err == nil {
// The default network causes dnsmasq to run and bind to all interfaces,
// including ours. This prevents discoverd from binding its DNS server.
// We don't use it, so destroy it if it exists.
defaultNet.Destroy()
}
// enable IP forwarding
if err := ioutil.WriteFile("/proc/sys/net/ipv4/ip_forward", []byte("1\n"), 0644); err != nil {
return err
}
// Set up iptables for outbound traffic masquerading from containers to the
// rest of the network.
if err := iptables.EnableOutboundNAT(bridgeName, l.bridgeNet.String()); err != nil {
return err
}
// Read DNS config, discoverd uses the nameservers
dnsConf, err := dns.ClientConfigFromFile("/etc/resolv.conf")
if err != nil {
return err
}
config.Resolvers = dnsConf.Servers
// Write a resolv.conf to be bind-mounted into containers pointing at the
// future discoverd DNS listener
if err := os.MkdirAll("/etc/flynn", 0755); err != nil {
return err
}
var resolvSearch string
if len(dnsConf.Search) > 0 {
resolvSearch = fmt.Sprintf("search %s\n", strings.Join(dnsConf.Search, " "))
}
if err := ioutil.WriteFile("/etc/flynn/resolv.conf", []byte(fmt.Sprintf("%snameserver %s\n", resolvSearch, l.bridgeAddr.String())), 0644); err != nil {
return err
}
l.resolvConf = "/etc/flynn/resolv.conf"
// Allocate IPs for running jobs
for i, container := range l.containers {
if !container.job.Config.HostNetwork {
var err error
l.containers[i].IP, err = l.ipalloc.RequestIP(l.bridgeNet, container.IP)
if err != nil {
grohl.Log(grohl.Data{"fn": "ConfigureNetworking", "at": "request_ip", "status": "error", "err": err})
}
}
}
close(l.networkConfigured)
return nil
}
func NewLibvirtLXCBackend(state *State, portAlloc map[string]*ports.Allocator, volPath, logPath, initPath string) (Backend, error) {
libvirtc, err := libvirt.NewVirConnection("lxc:///")
if err != nil {
return nil, err
}
pinkertonCtx, err := pinkerton.BuildContext("aufs", "/var/lib/docker")
if err != nil {
return nil, err
}
if err := writeResolvConf("/etc/flynn/resolv.conf"); err != nil {
return nil, fmt.Errorf("Could not create resolv.conf: %s", err)
}
b := random.Bytes(5)
bridgeMAC := fmt.Sprintf("fe:%02x:%02x:%02x:%02x:%02x", b[0], b[1], b[2], b[3], b[4])
network, err := libvirtc.LookupNetworkByName(libvirtNetName)
if err != nil {
n := <.Network{
Name: libvirtNetName,
Bridge: lt.Bridge{Name: bridgeName, STP: "off"},
IP: lt.IP{Address: bridgeAddr.String(), Netmask: bridgeMask},
MAC: lt.MAC{Address: bridgeMAC},
}
network, err = libvirtc.NetworkDefineXML(string(n.XML()))
if err != nil {
return nil, err
}
}
active, err := network.IsActive()
if err != nil {
return nil, err
}
if !active {
if err := network.Create(); err != nil {
return nil, err
}
}
// We need to explicitly assign the MAC address to avoid it changing to a lower value
// See: https://github.com/flynn/flynn/issues/223
if err := netlink.NetworkSetMacAddress(bridgeName, bridgeMAC); err != nil {
return nil, err
}
iptables.RemoveExistingChain("FLYNN", bridgeName)
chain, err := iptables.NewChain("FLYNN", bridgeName)
if err != nil {
return nil, err
}
if err := ioutil.WriteFile("/proc/sys/net/ipv4/conf/"+bridgeName+"/route_localnet", []byte("1"), 0666); err != nil {
return nil, err
}
return &LibvirtLXCBackend{
LogPath: logPath,
VolPath: volPath,
InitPath: initPath,
libvirt: libvirtc,
state: state,
ports: portAlloc,
pinkerton: pinkertonCtx,
forwarder: ports.NewForwarder(net.ParseIP("0.0.0.0"), chain),
logs: make(map[string]*logbuf.Log),
containers: make(map[string]*libvirtContainer),
}, nil
}