func createVethPair() (string, string, error) {
defer osl.InitOSContext()()
// Generate a name for what will be the host side pipe interface
name1, err := netutils.GenerateIfaceName(vethPrefix, vethLen)
if err != nil {
return "", "", fmt.Errorf("error generating veth name1: %v", err)
}
// Generate a name for what will be the sandbox side pipe interface
name2, err := netutils.GenerateIfaceName(vethPrefix, vethLen)
if err != nil {
return "", "", fmt.Errorf("error generating veth name2: %v", err)
}
// Generate and add the interface pipe host <-> sandbox
veth := &netlink.Veth{
LinkAttrs: netlink.LinkAttrs{Name: name1, TxQLen: 0},
PeerName: name2}
if err := netlink.LinkAdd(veth); err != nil {
return "", "", fmt.Errorf("error creating veth pair: %v", err)
}
return name1, name2, nil
}
func (n *network) initSubnetSandbox(s *subnet) error {
// create a bridge and vxlan device for this subnet and move it to the sandbox
brName, err := netutils.GenerateIfaceName("bridge", 7)
if err != nil {
return err
}
sbox := n.sandbox()
if err := sbox.AddInterface(brName, "br",
sbox.InterfaceOptions().Address(s.gwIP),
sbox.InterfaceOptions().Bridge(true)); err != nil {
return fmt.Errorf("bridge creation in sandbox failed for subnet %q: %v", s.subnetIP.IP.String(), err)
}
vxlanName, err := createVxlan(n.vxlanID(s))
if err != nil {
return err
}
if err := sbox.AddInterface(vxlanName, "vxlan",
sbox.InterfaceOptions().Master(brName)); err != nil {
return fmt.Errorf("vxlan interface creation failed for subnet %q: %v", s.subnetIP.IP.String(), err)
}
n.Lock()
s.vxlanName = vxlanName
s.brName = brName
n.Unlock()
return nil
}
// creates a veth pair and adds it to a bridge
func CreateVethPair(iface string) (local string, guest string, err error) {
var (
vethPrefix = "veth"
vethLen = 7
)
// get the link of the iface we passed so we can use its MTU
brLink, err := netlink.LinkByName(iface)
if err != nil {
return "", "", fmt.Errorf("finding link with name %s failed: %v", iface, err)
}
local, err = netutils.GenerateIfaceName(vethPrefix, vethLen)
if err != nil {
return "", "", fmt.Errorf("error generating veth name: %v", err)
}
guest, err = netutils.GenerateIfaceName(vethPrefix, vethLen)
if err != nil {
return "", "", fmt.Errorf("error generating veth name: %v", err)
}
veth := &netlink.Veth{
LinkAttrs: netlink.LinkAttrs{Name: local, TxQLen: 0, MTU: brLink.Attrs().MTU},
PeerName: guest}
if err := netlink.LinkAdd(veth); err != nil {
return "", "", fmt.Errorf("error creating veth pair: %v", err)
}
exists, err := portExists(iface, local)
if err != nil {
return "", "", err
}
if !exists {
if err := portAdd(iface, local); err != nil {
return "", "", err
}
}
return local, guest, nil
}
func createVxlan(vni uint32) (string, error) {
name, err := netutils.GenerateIfaceName("vxlan", 7)
if err != nil {
return "", fmt.Errorf("error generating vxlan name: %v", err)
}
vxlan := &netlink.Vxlan{
LinkAttrs: netlink.LinkAttrs{Name: name},
VxlanId: int(vni),
Learning: true,
Proxy: true,
L3miss: true,
L2miss: true,
}
if err := netlink.LinkAdd(vxlan); err != nil {
return "", fmt.Errorf("error creating vxlan interface: %v", err)
}
return name, nil
}
func createVxlan(vni uint32) (string, error) {
defer osl.InitOSContext()()
name, err := netutils.GenerateIfaceName("vxlan", 7)
if err != nil {
return "", fmt.Errorf("error generating vxlan name: %v", err)
}
vxlan := &netlink.Vxlan{
LinkAttrs: netlink.LinkAttrs{Name: name},
VxlanId: int(vni),
Learning: true,
Port: int(nl.Swap16(vxlanPort)), //network endian order
Proxy: true,
L3miss: true,
L2miss: true,
}
if err := netlink.LinkAdd(vxlan); err != nil {
return "", fmt.Errorf("error creating vxlan interface: %v", err)
}
return name, nil
}
func (d *driver) CreateEndpoint(nid, eid types.UUID, epInfo driverapi.EndpointInfo, epOptions map[string]interface{}) error {
var (
ipv6Addr *net.IPNet
err error
)
defer sandbox.InitOSContext()()
if epInfo == nil {
return errors.New("invalid endpoint info passed")
}
if len(epInfo.Interfaces()) != 0 {
return errors.New("non empty interface list passed to bridge(local) driver")
}
// Get the network handler and make sure it exists
d.Lock()
n, ok := d.networks[nid]
d.Unlock()
if !ok {
return types.NotFoundErrorf("network %s does not exist", nid)
}
if n == nil {
return driverapi.ErrNoNetwork(nid)
}
// Sanity check
n.Lock()
if n.id != nid {
n.Unlock()
return InvalidNetworkIDError(nid)
}
n.Unlock()
// Check if endpoint id is good and retrieve correspondent endpoint
ep, err := n.getEndpoint(eid)
if err != nil {
return err
}
// Endpoint with that id exists either on desired or other sandbox
if ep != nil {
return driverapi.ErrEndpointExists(eid)
}
// Try to convert the options to endpoint configuration
epConfig, err := parseEndpointOptions(epOptions)
if err != nil {
return err
}
// Create and add the endpoint
n.Lock()
endpoint := &bridgeEndpoint{id: eid, config: epConfig}
n.endpoints[eid] = endpoint
n.Unlock()
// On failure make sure to remove the endpoint
defer func() {
if err != nil {
n.Lock()
delete(n.endpoints, eid)
n.Unlock()
}
}()
// Generate a name for what will be the host side pipe interface
hostIfName, err := netutils.GenerateIfaceName(vethPrefix, vethLen)
if err != nil {
return err
}
// Generate a name for what will be the sandbox side pipe interface
containerIfName, err := netutils.GenerateIfaceName(vethPrefix, vethLen)
if err != nil {
return err
}
// Generate and add the interface pipe host <-> sandbox
veth := &netlink.Veth{
LinkAttrs: netlink.LinkAttrs{Name: hostIfName, TxQLen: 0},
PeerName: containerIfName}
if err = netlink.LinkAdd(veth); err != nil {
return err
}
// Get the host side pipe interface handler
host, err := netlink.LinkByName(hostIfName)
if err != nil {
return err
}
defer func() {
if err != nil {
netlink.LinkDel(host)
}
}()
// Get the sandbox side pipe interface handler
sbox, err := netlink.LinkByName(containerIfName)
if err != nil {
return err
}
defer func() {
if err != nil {
netlink.LinkDel(sbox)
}
}()
n.Lock()
config := n.config
n.Unlock()
// Add bridge inherited attributes to pipe interfaces
if config.Mtu != 0 {
err = netlink.LinkSetMTU(host, config.Mtu)
if err != nil {
return err
}
err = netlink.LinkSetMTU(sbox, config.Mtu)
if err != nil {
return err
}
}
// Attach host side pipe interface into the bridge
if err = addToBridge(hostIfName, config.BridgeName); err != nil {
return fmt.Errorf("adding interface %s to bridge %s failed: %v", hostIfName, config.BridgeName, err)
}
if !config.EnableUserlandProxy {
err = setHairpinMode(host, true)
if err != nil {
return err
}
}
// v4 address for the sandbox side pipe interface
ip4, err := ipAllocator.RequestIP(n.bridge.bridgeIPv4, nil)
if err != nil {
return err
}
ipv4Addr := &net.IPNet{IP: ip4, Mask: n.bridge.bridgeIPv4.Mask}
// Down the interface before configuring mac address.
if err = netlink.LinkSetDown(sbox); err != nil {
return fmt.Errorf("could not set link down for container interface %s: %v", containerIfName, err)
}
// Set the sbox's MAC. If specified, use the one configured by user, otherwise generate one based on IP.
mac := electMacAddress(epConfig, ip4)
err = netlink.LinkSetHardwareAddr(sbox, mac)
if err != nil {
return fmt.Errorf("could not set mac address for container interface %s: %v", containerIfName, err)
}
endpoint.macAddress = mac
// Up the host interface after finishing all netlink configuration
if err = netlink.LinkSetUp(host); err != nil {
return fmt.Errorf("could not set link up for host interface %s: %v", hostIfName, err)
}
// v6 address for the sandbox side pipe interface
ipv6Addr = &net.IPNet{}
if config.EnableIPv6 {
var ip6 net.IP
network := n.bridge.bridgeIPv6
if config.FixedCIDRv6 != nil {
network = config.FixedCIDRv6
}
ones, _ := network.Mask.Size()
if ones <= 80 {
ip6 = make(net.IP, len(network.IP))
copy(ip6, network.IP)
for i, h := range mac {
ip6[i+10] = h
}
}
ip6, err := ipAllocator.RequestIP(network, ip6)
if err != nil {
return err
}
ipv6Addr = &net.IPNet{IP: ip6, Mask: network.Mask}
}
// Create the sandbox side pipe interface
endpoint.srcName = containerIfName
endpoint.addr = ipv4Addr
if config.EnableIPv6 {
endpoint.addrv6 = ipv6Addr
}
err = epInfo.AddInterface(ifaceID, endpoint.macAddress, *ipv4Addr, *ipv6Addr)
if err != nil {
return err
}
// Program any required port mapping and store them in the endpoint
endpoint.portMapping, err = n.allocatePorts(epConfig, endpoint, config.DefaultBindingIP, config.EnableUserlandProxy)
if err != nil {
return err
}
return nil
}
func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, epOptions map[string]interface{}) error {
defer osl.InitOSContext()()
if ifInfo == nil {
return errors.New("invalid interface info passed")
}
// Get the network handler and make sure it exists
d.Lock()
n, ok := d.networks[nid]
dconfig := d.config
d.Unlock()
if !ok {
return types.NotFoundErrorf("network %s does not exist", nid)
}
if n == nil {
return driverapi.ErrNoNetwork(nid)
}
// Sanity check
n.Lock()
if n.id != nid {
n.Unlock()
return InvalidNetworkIDError(nid)
}
n.Unlock()
// Check if endpoint id is good and retrieve correspondent endpoint
ep, err := n.getEndpoint(eid)
if err != nil {
return err
}
// Endpoint with that id exists either on desired or other sandbox
if ep != nil {
return driverapi.ErrEndpointExists(eid)
}
// Try to convert the options to endpoint configuration
epConfig, err := parseEndpointOptions(epOptions)
if err != nil {
return err
}
// Create and add the endpoint
n.Lock()
endpoint := &bridgeEndpoint{id: eid, config: epConfig}
n.endpoints[eid] = endpoint
n.Unlock()
// On failure make sure to remove the endpoint
defer func() {
if err != nil {
n.Lock()
delete(n.endpoints, eid)
n.Unlock()
}
}()
// Generate a name for what will be the host side pipe interface
hostIfName, err := netutils.GenerateIfaceName(vethPrefix, vethLen)
if err != nil {
return err
}
// Generate a name for what will be the sandbox side pipe interface
containerIfName, err := netutils.GenerateIfaceName(vethPrefix, vethLen)
if err != nil {
return err
}
// Generate and add the interface pipe host <-> sandbox
veth := &netlink.Veth{
LinkAttrs: netlink.LinkAttrs{Name: hostIfName, TxQLen: 0},
PeerName: containerIfName}
if err = netlink.LinkAdd(veth); err != nil {
return types.InternalErrorf("failed to add the host (%s) <=> sandbox (%s) pair interfaces: %v", hostIfName, containerIfName, err)
}
// Get the host side pipe interface handler
host, err := netlink.LinkByName(hostIfName)
if err != nil {
return types.InternalErrorf("failed to find host side interface %s: %v", hostIfName, err)
}
defer func() {
if err != nil {
netlink.LinkDel(host)
}
}()
// Get the sandbox side pipe interface handler
sbox, err := netlink.LinkByName(containerIfName)
if err != nil {
return types.InternalErrorf("failed to find sandbox side interface %s: %v", containerIfName, err)
}
defer func() {
if err != nil {
netlink.LinkDel(sbox)
}
}()
n.Lock()
config := n.config
n.Unlock()
// Add bridge inherited attributes to pipe interfaces
if config.Mtu != 0 {
err = netlink.LinkSetMTU(host, config.Mtu)
if err != nil {
return types.InternalErrorf("failed to set MTU on host interface %s: %v", hostIfName, err)
}
err = netlink.LinkSetMTU(sbox, config.Mtu)
if err != nil {
return types.InternalErrorf("failed to set MTU on sandbox interface %s: %v", containerIfName, err)
}
}
// Attach host side pipe interface into the bridge
if err = addToBridge(hostIfName, config.BridgeName); err != nil {
return fmt.Errorf("adding interface %s to bridge %s failed: %v", hostIfName, config.BridgeName, err)
}
if !dconfig.EnableUserlandProxy {
err = setHairpinMode(host, true)
if err != nil {
return err
}
}
// Store the sandbox side pipe interface parameters
endpoint.srcName = containerIfName
endpoint.macAddress = ifInfo.MacAddress()
endpoint.addr = ifInfo.Address()
endpoint.addrv6 = ifInfo.AddressIPv6()
// Set the sbox's MAC if not provided. If specified, use the one configured by user, otherwise generate one based on IP.
if endpoint.macAddress == nil {
endpoint.macAddress = electMacAddress(epConfig, endpoint.addr.IP)
if err = ifInfo.SetMacAddress(endpoint.macAddress); err != nil {
return err
}
}
// Up the host interface after finishing all netlink configuration
if err = netlink.LinkSetUp(host); err != nil {
return fmt.Errorf("could not set link up for host interface %s: %v", hostIfName, err)
}
if endpoint.addrv6 == nil && config.EnableIPv6 {
var ip6 net.IP
network := n.bridge.bridgeIPv6
if config.AddressIPv6 != nil {
network = config.AddressIPv6
}
ones, _ := network.Mask.Size()
if ones > 80 {
err = types.ForbiddenErrorf("Cannot self generate an IPv6 address on network %v: At least 48 host bits are needed.", network)
return err
}
ip6 = make(net.IP, len(network.IP))
copy(ip6, network.IP)
for i, h := range endpoint.macAddress {
ip6[i+10] = h
}
endpoint.addrv6 = &net.IPNet{IP: ip6, Mask: network.Mask}
if err = ifInfo.SetIPAddress(endpoint.addrv6); err != nil {
return err
}
}
return nil
}
func (l *LibcontainerBackend) Run(job *host.Job, runConfig *RunConfig, rateLimitBucket *RateLimitBucket) (err error) {
log := l.logger.New("fn", "run", "job.id", job.ID)
// if the job has been stopped, just return
if l.state.GetJob(job.ID).ForceStop {
log.Info("skipping start of stopped job")
return nil
}
log.Info("starting job", "job.artifact.uri", job.ImageArtifact.URI, "job.args", job.Config.Args)
defer func() {
if err != nil {
l.state.SetStatusFailed(job.ID, err)
}
}()
if job.Partition == "" {
job.Partition = defaultPartition
}
if _, ok := l.partitionCGroups[job.Partition]; !ok {
return fmt.Errorf("host: invalid job partition %q", job.Partition)
}
wait := func(ch chan struct{}) {
if rateLimitBucket != nil {
// unblock the rate limiter whilst waiting
rateLimitBucket.Put()
defer rateLimitBucket.Wait()
}
<-ch
}
if !job.Config.HostNetwork {
wait(l.networkConfigured)
}
if _, ok := job.Config.Env["DISCOVERD"]; !ok {
wait(l.discoverdConfigured)
}
if runConfig == nil {
runConfig = &RunConfig{}
}
container := &Container{
ID: job.ID,
l: l,
job: job,
done: make(chan struct{}),
}
if !job.Config.HostNetwork {
container.IP, err = l.ipalloc.RequestIP(l.bridgeNet, runConfig.IP)
if err != nil {
log.Error("error requesting ip", "err", err)
return err
}
log.Info("obtained ip", "network", l.bridgeNet.String(), "ip", container.IP.String())
l.state.SetContainerIP(job.ID, container.IP)
}
defer func() {
if err != nil {
go container.cleanup()
}
}()
log.Info("pulling image")
artifactURI, err := l.resolveDiscoverdURI(job.ImageArtifact.URI)
if err != nil {
log.Error("error resolving artifact URI", "err", err)
return err
}
// TODO(lmars): stream pull progress (maybe to the app log?)
imageID, err := l.pinkerton.PullDocker(artifactURI, ioutil.Discard)
if err != nil {
log.Error("error pulling image", "err", err)
return err
}
log.Info("reading image config")
imageConfig, err := readDockerImageConfig(imageID)
if err != nil {
log.Error("error reading image config", "err", err)
return err
}
log.Info("checking out image")
var rootPath string
// creating an AUFS mount can fail intermittently with EINVAL, so try a
// few times (see https://github.com/flynn/flynn/issues/2044)
for start := time.Now(); time.Since(start) < time.Second; time.Sleep(50 * time.Millisecond) {
rootPath, err = l.pinkerton.Checkout(job.ID, imageID)
if err == nil || !strings.HasSuffix(err.Error(), "invalid argument") {
break
}
}
if err != nil {
log.Error("error checking out image", "err", err)
return err
}
container.RootPath = rootPath
config := &configs.Config{
Rootfs: rootPath,
Capabilities: defaultCapabilities,
Namespaces: configs.Namespaces([]configs.Namespace{
{Type: configs.NEWNS},
{Type: configs.NEWUTS},
{Type: configs.NEWIPC},
{Type: configs.NEWPID},
}),
Cgroups: &configs.Cgroup{
Path: filepath.Join("/flynn", job.Partition, job.ID),
Resources: &configs.Resources{
AllowedDevices: configs.DefaultAllowedDevices,
Memory: defaultMemory,
},
},
MaskPaths: []string{
"/proc/kcore",
},
ReadonlyPaths: []string{
"/proc/sys", "/proc/sysrq-trigger", "/proc/irq", "/proc/bus",
},
Devices: configs.DefaultAutoCreatedDevices,
Mounts: []*configs.Mount{
{
Source: "proc",
Destination: "/proc",
Device: "proc",
Flags: defaultMountFlags,
},
{
Source: "sysfs",
Destination: "/sys",
Device: "sysfs",
Flags: defaultMountFlags | syscall.MS_RDONLY,
},
{
Source: "tmpfs",
Destination: "/dev",
Device: "tmpfs",
Flags: syscall.MS_NOSUID | syscall.MS_STRICTATIME,
Data: "mode=755",
},
{
Source: "devpts",
Destination: "/dev/pts",
Device: "devpts",
Flags: syscall.MS_NOSUID | syscall.MS_NOEXEC,
Data: "newinstance,ptmxmode=0666,mode=0620,gid=5",
},
{
Device: "tmpfs",
Source: "shm",
Destination: "/dev/shm",
Data: "mode=1777,size=65536k",
Flags: defaultMountFlags,
},
{
Destination: "/sys/fs/cgroup",
Device: "cgroup",
Flags: defaultMountFlags | syscall.MS_RDONLY,
},
},
}
if spec, ok := job.Resources[resource.TypeMaxFD]; ok && spec.Limit != nil && spec.Request != nil {
log.Info(fmt.Sprintf("setting max fd limit to %d / %d", *spec.Request, *spec.Limit))
config.Rlimits = append(config.Rlimits, configs.Rlimit{
Type: syscall.RLIMIT_NOFILE,
Hard: uint64(*spec.Limit),
Soft: uint64(*spec.Request),
})
}
if spec, ok := job.Resources[resource.TypeMaxProcs]; ok && spec.Limit != nil && spec.Request != nil {
log.Info(fmt.Sprintf("setting max processes limit to %d / %d", *spec.Request, *spec.Limit))
config.Rlimits = append(config.Rlimits, configs.Rlimit{
Type: RLIMIT_NPROC,
Hard: uint64(*spec.Limit),
Soft: uint64(*spec.Request),
})
}
log.Info("mounting container directories and files")
jobIDParts := strings.SplitN(job.ID, "-", 2)
var hostname string
if len(jobIDParts) == 1 {
hostname = jobIDParts[0]
} else {
hostname = jobIDParts[1]
}
if len(hostname) > 64 {
hostname = hostname[:64]
}
if err := os.MkdirAll(filepath.Join(rootPath, "etc"), 0755); err != nil {
log.Error("error creating /etc in container root", "err", err)
return err
}
if err := writeHostname(filepath.Join(rootPath, "etc/hosts"), hostname); err != nil {
log.Error("error writing hosts file", "err", err)
return err
}
if err := os.MkdirAll(filepath.Join(rootPath, ".container-shared"), 0700); err != nil {
log.Error("error createing .container-shared", "err", err)
return err
}
addBindMount(config, l.InitPath, "/.containerinit", false)
addBindMount(config, l.resolvConf, "/etc/resolv.conf", false)
for _, m := range job.Config.Mounts {
if m.Target == "" {
return errors.New("host: invalid empty mount target")
}
addBindMount(config, m.Target, m.Location, m.Writeable)
}
// apply volumes
for _, v := range job.Config.Volumes {
vol := l.vman.GetVolume(v.VolumeID)
if vol == nil {
err := fmt.Errorf("job %s required volume %s, but that volume does not exist", job.ID, v.VolumeID)
log.Error("missing required volume", "volumeID", v.VolumeID, "err", err)
return err
}
addBindMount(config, vol.Location(), v.Target, v.Writeable)
}
// mutating job state, take state write lock
l.state.mtx.Lock()
if job.Config.Env == nil {
job.Config.Env = make(map[string]string)
}
for i, p := range job.Config.Ports {
if p.Proto != "tcp" && p.Proto != "udp" {
err := fmt.Errorf("unknown port proto %q", p.Proto)
log.Error("error allocating port", "proto", p.Proto, "err", err)
return err
}
if p.Port == 0 {
job.Config.Ports[i].Port = 5000 + i
}
if i == 0 {
job.Config.Env["PORT"] = strconv.Itoa(job.Config.Ports[i].Port)
}
job.Config.Env[fmt.Sprintf("PORT_%d", i)] = strconv.Itoa(job.Config.Ports[i].Port)
}
if !job.Config.HostNetwork {
job.Config.Env["EXTERNAL_IP"] = container.IP.String()
}
// release the write lock, we won't mutate global structures from here on out
l.state.mtx.Unlock()
initConfig := &containerinit.Config{
Args: job.Config.Args,
TTY: job.Config.TTY,
OpenStdin: job.Config.Stdin,
WorkDir: job.Config.WorkingDir,
Resources: job.Resources,
FileArtifacts: job.FileArtifacts,
}
if !job.Config.HostNetwork {
initConfig.IP = container.IP.String() + "/24"
initConfig.Gateway = l.bridgeAddr.String()
}
if initConfig.WorkDir == "" {
initConfig.WorkDir = imageConfig.WorkingDir
}
if job.Config.Uid > 0 {
initConfig.User = strconv.Itoa(job.Config.Uid)
} else if imageConfig.User != "" {
// TODO: check and lookup user from image config
}
if len(job.Config.Args) == 0 {
initConfig.Args = append(imageConfig.Entrypoint, imageConfig.Cmd...)
}
for _, port := range job.Config.Ports {
initConfig.Ports = append(initConfig.Ports, port)
}
log.Info("writing config")
l.envMtx.RLock()
err = writeContainerConfig(filepath.Join(rootPath, ".containerconfig"), initConfig,
map[string]string{
"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"TERM": "xterm",
"HOME": "/",
},
l.defaultEnv,
job.Config.Env,
map[string]string{
"HOSTNAME": hostname,
},
)
l.envMtx.RUnlock()
if err != nil {
log.Error("error writing config", "err", err)
return err
}
if job.Config.HostNetwork {
// allow host network jobs to configure the network
config.Capabilities = append(config.Capabilities, "CAP_NET_ADMIN")
} else {
ifaceName, err := netutils.GenerateIfaceName("veth", 4)
if err != nil {
return err
}
config.Hostname = hostname
config.Namespaces = append(config.Namespaces, configs.Namespace{Type: configs.NEWNET})
config.Networks = []*configs.Network{
{
Type: "loopback",
Address: "127.0.0.1/0",
Gateway: "localhost",
},
{
Type: "veth",
Name: "eth0",
Bridge: l.bridgeName,
Address: initConfig.IP,
Gateway: initConfig.Gateway,
Mtu: 1500,
HostInterfaceName: ifaceName,
},
}
}
if spec, ok := job.Resources[resource.TypeMemory]; ok && spec.Limit != nil {
config.Cgroups.Resources.Memory = *spec.Limit
}
if spec, ok := job.Resources[resource.TypeCPU]; ok && spec.Limit != nil {
config.Cgroups.Resources.CpuShares = milliCPUToShares(*spec.Limit)
}
c, err := l.factory.Create(job.ID, config)
if err != nil {
return err
}
process := &libcontainer.Process{
Args: []string{"/.containerinit", job.ID},
User: "******",
}
if err := c.Run(process); err != nil {
c.Destroy()
return err
}
// TODO: detach? an update will detach all container anyway
go process.Wait()
container.container = c
// TODO: still necessary?
l.state.SetContainerID(job.ID, job.ID)
go container.watch(nil, nil)
log.Info("job started")
return nil
}
// Join method is invoked when a Sandbox is attached to an endpoint.
func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error {
defer osl.InitOSContext()()
n, err := d.getNetwork(nid)
if err != nil {
return err
}
endpoint := n.endpoint(eid)
if endpoint == nil {
return fmt.Errorf("could not find endpoint with id %s", eid)
}
// generate a name for the iface that will be renamed to eth0 in the sbox
containerIfName, err := netutils.GenerateIfaceName(ns.NlHandle(), vethPrefix, vethLen)
if err != nil {
return fmt.Errorf("error generating an interface name: %s", err)
}
// create the netlink macvlan interface
vethName, err := createMacVlan(containerIfName, n.config.Parent, n.config.MacvlanMode)
if err != nil {
return err
}
// bind the generated iface name to the endpoint
endpoint.srcName = vethName
ep := n.endpoint(eid)
if ep == nil {
return fmt.Errorf("could not find endpoint with id %s", eid)
}
// parse and match the endpoint address with the available v4 subnets
if len(n.config.Ipv4Subnets) > 0 {
s := n.getSubnetforIPv4(ep.addr)
if s == nil {
return fmt.Errorf("could not find a valid ipv4 subnet for endpoint %s", eid)
}
v4gw, _, err := net.ParseCIDR(s.GwIP)
if err != nil {
return fmt.Errorf("gatway %s is not a valid ipv4 address: %v", s.GwIP, err)
}
err = jinfo.SetGateway(v4gw)
if err != nil {
return err
}
logrus.Debugf("Macvlan Endpoint Joined with IPv4_Addr: %s, Gateway: %s, MacVlan_Mode: %s, Parent: %s",
ep.addr.IP.String(), v4gw.String(), n.config.MacvlanMode, n.config.Parent)
}
// parse and match the endpoint address with the available v6 subnets
if len(n.config.Ipv6Subnets) > 0 {
s := n.getSubnetforIPv6(ep.addrv6)
if s == nil {
return fmt.Errorf("could not find a valid ipv6 subnet for endpoint %s", eid)
}
v6gw, _, err := net.ParseCIDR(s.GwIP)
if err != nil {
return fmt.Errorf("gatway %s is not a valid ipv6 address: %v", s.GwIP, err)
}
err = jinfo.SetGatewayIPv6(v6gw)
if err != nil {
return err
}
logrus.Debugf("Macvlan Endpoint Joined with IPv6_Addr: %s Gateway: %s MacVlan_Mode: %s, Parent: %s",
ep.addrv6.IP.String(), v6gw.String(), n.config.MacvlanMode, n.config.Parent)
}
iNames := jinfo.InterfaceName()
err = iNames.SetNames(vethName, containerVethPrefix)
if err != nil {
return err
}
return nil
}
// Join method is invoked when a Sandbox is attached to an endpoint.
func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error {
defer osl.InitOSContext()()
n, err := d.getNetwork(nid)
if err != nil {
return err
}
endpoint := n.endpoint(eid)
if endpoint == nil {
return fmt.Errorf("could not find endpoint with id %s", eid)
}
// generate a name for the iface that will be renamed to eth0 in the sbox
containerIfName, err := netutils.GenerateIfaceName(ns.NlHandle(), vethPrefix, vethLen)
if err != nil {
return fmt.Errorf("error generating an interface name: %v", err)
}
// create the netlink ipvlan interface
vethName, err := createIPVlan(containerIfName, n.config.Parent, n.config.IpvlanMode)
if err != nil {
return err
}
// bind the generated iface name to the endpoint
endpoint.srcName = vethName
ep := n.endpoint(eid)
if ep == nil {
return fmt.Errorf("could not find endpoint with id %s", eid)
}
if n.config.IpvlanMode == modeL3 {
// disable gateway services to add a default gw using dev eth0 only
jinfo.DisableGatewayService()
defaultRoute, err := ifaceGateway(defaultV4RouteCidr)
if err != nil {
return err
}
if err := jinfo.AddStaticRoute(defaultRoute.Destination, defaultRoute.RouteType, defaultRoute.NextHop); err != nil {
return fmt.Errorf("failed to set an ipvlan l3 mode ipv4 default gateway: %v", err)
}
logrus.Debugf("Ipvlan Endpoint Joined with IPv4_Addr: %s, Ipvlan_Mode: %s, Parent: %s",
ep.addr.IP.String(), n.config.IpvlanMode, n.config.Parent)
// If the endpoint has a v6 address, set a v6 default route
if ep.addrv6 != nil {
default6Route, err := ifaceGateway(defaultV6RouteCidr)
if err != nil {
return err
}
if err = jinfo.AddStaticRoute(default6Route.Destination, default6Route.RouteType, default6Route.NextHop); err != nil {
return fmt.Errorf("failed to set an ipvlan l3 mode ipv6 default gateway: %v", err)
}
logrus.Debugf("Ipvlan Endpoint Joined with IPv6_Addr: %s, Ipvlan_Mode: %s, Parent: %s",
ep.addrv6.IP.String(), n.config.IpvlanMode, n.config.Parent)
}
}
if n.config.IpvlanMode == modeL2 {
// parse and correlate the endpoint v4 address with the available v4 subnets
if len(n.config.Ipv4Subnets) > 0 {
s := n.getSubnetforIPv4(ep.addr)
if s == nil {
return fmt.Errorf("could not find a valid ipv4 subnet for endpoint %s", eid)
}
v4gw, _, err := net.ParseCIDR(s.GwIP)
if err != nil {
return fmt.Errorf("gatway %s is not a valid ipv4 address: %v", s.GwIP, err)
}
err = jinfo.SetGateway(v4gw)
if err != nil {
return err
}
logrus.Debugf("Ipvlan Endpoint Joined with IPv4_Addr: %s, Gateway: %s, Ipvlan_Mode: %s, Parent: %s",
ep.addr.IP.String(), v4gw.String(), n.config.IpvlanMode, n.config.Parent)
}
// parse and correlate the endpoint v6 address with the available v6 subnets
if len(n.config.Ipv6Subnets) > 0 {
s := n.getSubnetforIPv6(ep.addrv6)
if s == nil {
return fmt.Errorf("could not find a valid ipv6 subnet for endpoint %s", eid)
}
v6gw, _, err := net.ParseCIDR(s.GwIP)
if err != nil {
return fmt.Errorf("gatway %s is not a valid ipv6 address: %v", s.GwIP, err)
}
err = jinfo.SetGatewayIPv6(v6gw)
if err != nil {
return err
}
logrus.Debugf("Ipvlan Endpoint Joined with IPv6_Addr: %s, Gateway: %s, Ipvlan_Mode: %s, Parent: %s",
ep.addrv6.IP.String(), v6gw.String(), n.config.IpvlanMode, n.config.Parent)
}
}
iNames := jinfo.InterfaceName()
err = iNames.SetNames(vethName, containerVethPrefix)
if err != nil {
return err
}
if err = d.storeUpdate(ep); err != nil {
return fmt.Errorf("failed to save ipvlan endpoint %s to store: %v", ep.id[0:7], err)
}
return nil
}
func (l *LibcontainerBackend) Run(job *host.Job, runConfig *RunConfig, rateLimitBucket *RateLimitBucket) (err error) {
log := l.Logger.New("fn", "run", "job.id", job.ID)
// if the job has been stopped, just return
if l.State.GetJob(job.ID).ForceStop {
log.Info("skipping start of stopped job")
return nil
}
log.Info("starting job", "job.args", job.Config.Args)
defer func() {
if err != nil {
l.State.SetStatusFailed(job.ID, err)
}
}()
if job.Partition == "" {
job.Partition = defaultPartition
}
if _, ok := l.PartitionCGroups[job.Partition]; !ok {
return fmt.Errorf("host: invalid job partition %q", job.Partition)
}
wait := func(ch chan struct{}) {
if rateLimitBucket != nil {
// unblock the rate limiter whilst waiting
rateLimitBucket.Put()
defer rateLimitBucket.Wait()
}
<-ch
}
if !job.Config.HostNetwork {
wait(l.networkConfigured)
}
if _, ok := job.Config.Env["DISCOVERD"]; !ok {
wait(l.discoverdConfigured)
}
if runConfig == nil {
runConfig = &RunConfig{}
}
container := &Container{
ID: job.ID,
MuxConfig: &logmux.Config{
AppID: job.Metadata["flynn-controller.app"],
HostID: l.State.id,
JobType: job.Metadata["flynn-controller.type"],
JobID: job.ID,
},
l: l,
job: job,
done: make(chan struct{}),
}
if !job.Config.HostNetwork {
container.IP, err = l.ipalloc.RequestIP(l.bridgeNet, runConfig.IP)
if err != nil {
log.Error("error requesting ip", "err", err)
return err
}
log.Info("obtained ip", "network", l.bridgeNet.String(), "ip", container.IP.String())
l.State.SetContainerIP(job.ID, container.IP)
}
defer func() {
if err != nil {
go container.cleanup()
}
}()
log.Info("setting up rootfs")
rootPath := filepath.Join("/var/lib/flynn/image/mnt", job.ID)
tmpPath := filepath.Join("/var/lib/flynn/image/tmp", job.ID)
for _, path := range []string{rootPath, tmpPath} {
if err := os.MkdirAll(path, 0755); err != nil {
log.Error("error setting up rootfs", "err", err)
return err
}
}
rootMount, err := l.rootOverlayMount(job)
if err != nil {
log.Error("error setting up rootfs", "err", err)
return err
}
container.RootPath = rootPath
container.TmpPath = tmpPath
config := &configs.Config{
Rootfs: rootPath,
Capabilities: defaultCapabilities,
Namespaces: configs.Namespaces([]configs.Namespace{
{Type: configs.NEWNS},
{Type: configs.NEWUTS},
{Type: configs.NEWIPC},
{Type: configs.NEWPID},
}),
Cgroups: &configs.Cgroup{
Path: filepath.Join("/flynn", job.Partition, job.ID),
Resources: &configs.Resources{
AllowedDevices: configs.DefaultAllowedDevices,
Memory: defaultMemory,
},
},
MaskPaths: []string{
"/proc/kcore",
},
ReadonlyPaths: []string{
"/proc/sys", "/proc/sysrq-trigger", "/proc/irq", "/proc/bus",
},
Devices: configs.DefaultAutoCreatedDevices,
Mounts: append([]*configs.Mount{rootMount}, []*configs.Mount{
{
Source: "proc",
Destination: "/proc",
Device: "proc",
Flags: defaultMountFlags,
},
{
Source: "sysfs",
Destination: "/sys",
Device: "sysfs",
Flags: defaultMountFlags | syscall.MS_RDONLY,
},
{
Source: "tmpfs",
Destination: "/dev",
Device: "tmpfs",
Flags: syscall.MS_NOSUID | syscall.MS_STRICTATIME,
Data: "mode=755",
},
{
Source: "devpts",
Destination: "/dev/pts",
Device: "devpts",
Flags: syscall.MS_NOSUID | syscall.MS_NOEXEC,
Data: "newinstance,ptmxmode=0666,mode=0620,gid=5",
},
{
Device: "tmpfs",
Source: "shm",
Destination: "/dev/shm",
Data: "mode=1777,size=65536k",
Flags: defaultMountFlags,
},
{
Destination: "/sys/fs/cgroup",
Device: "cgroup",
Flags: defaultMountFlags | syscall.MS_RDONLY,
},
}...),
}
if spec, ok := job.Resources[resource.TypeMaxFD]; ok && spec.Limit != nil && spec.Request != nil {
log.Info(fmt.Sprintf("setting max fd limit to %d / %d", *spec.Request, *spec.Limit))
config.Rlimits = append(config.Rlimits, configs.Rlimit{
Type: syscall.RLIMIT_NOFILE,
Hard: uint64(*spec.Limit),
Soft: uint64(*spec.Request),
})
}
if spec, ok := job.Resources[resource.TypeMaxProcs]; ok && spec.Limit != nil && spec.Request != nil {
log.Info(fmt.Sprintf("setting max processes limit to %d / %d", *spec.Request, *spec.Limit))
config.Rlimits = append(config.Rlimits, configs.Rlimit{
Type: RLIMIT_NPROC,
Hard: uint64(*spec.Limit),
Soft: uint64(*spec.Request),
})
}
log.Info("mounting container directories and files")
jobIDParts := strings.SplitN(job.ID, "-", 2)
var hostname string
if len(jobIDParts) == 1 {
hostname = jobIDParts[0]
} else {
hostname = jobIDParts[1]
}
if len(hostname) > 64 {
hostname = hostname[:64]
}
if err := os.MkdirAll(filepath.Join(tmpPath, "etc"), 0755); err != nil {
log.Error("error creating container /etc", "err", err)
return err
}
etcHosts := filepath.Join(tmpPath, "etc/hosts")
if err := writeHostname(etcHosts, hostname); err != nil {
log.Error("error writing hosts file", "err", err)
return err
}
sharedDir := filepath.Join(tmpPath, ".container-shared")
if err := os.MkdirAll(sharedDir, 0700); err != nil {
log.Error("error creating .container-shared", "err", err)
return err
}
config.Mounts = append(config.Mounts,
bindMount(l.InitPath, "/.containerinit", false),
bindMount(l.resolvConf, "/etc/resolv.conf", false),
bindMount(etcHosts, "/etc/hosts", true),
bindMount(sharedDir, "/.container-shared", true),
)
for _, m := range job.Config.Mounts {
if m.Target == "" {
return errors.New("host: invalid empty mount target")
}
config.Mounts = append(config.Mounts, bindMount(m.Target, m.Location, m.Writeable))
}
// apply volumes
for _, v := range job.Config.Volumes {
vol := l.VolManager.GetVolume(v.VolumeID)
if vol == nil {
err := fmt.Errorf("job %s required volume %s, but that volume does not exist", job.ID, v.VolumeID)
log.Error("missing required volume", "volumeID", v.VolumeID, "err", err)
return err
}
config.Mounts = append(config.Mounts, bindMount(vol.Location(), v.Target, v.Writeable))
}
// mutating job state, take state write lock
l.State.mtx.Lock()
if job.Config.Env == nil {
job.Config.Env = make(map[string]string)
}
for i, p := range job.Config.Ports {
if p.Proto != "tcp" && p.Proto != "udp" {
err := fmt.Errorf("unknown port proto %q", p.Proto)
log.Error("error allocating port", "proto", p.Proto, "err", err)
return err
}
if p.Port == 0 {
job.Config.Ports[i].Port = 5000 + i
}
if i == 0 {
job.Config.Env["PORT"] = strconv.Itoa(job.Config.Ports[i].Port)
}
job.Config.Env[fmt.Sprintf("PORT_%d", i)] = strconv.Itoa(job.Config.Ports[i].Port)
}
if !job.Config.HostNetwork {
job.Config.Env["EXTERNAL_IP"] = container.IP.String()
}
// release the write lock, we won't mutate global structures from here on out
l.State.mtx.Unlock()
initConfig := &containerinit.Config{
Args: job.Config.Args,
TTY: job.Config.TTY,
OpenStdin: job.Config.Stdin,
WorkDir: job.Config.WorkingDir,
Uid: job.Config.Uid,
Gid: job.Config.Gid,
Resources: job.Resources,
LogLevel: l.InitLogLevel,
}
if !job.Config.HostNetwork {
initConfig.IP = container.IP.String() + "/24"
initConfig.Gateway = l.bridgeAddr.String()
}
for _, port := range job.Config.Ports {
initConfig.Ports = append(initConfig.Ports, port)
}
log.Info("writing config")
configPath := filepath.Join(tmpPath, ".containerconfig")
l.envMtx.RLock()
err = writeContainerConfig(configPath, initConfig,
map[string]string{
"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"TERM": "xterm",
"HOME": "/",
},
l.defaultEnv,
job.Config.Env,
map[string]string{
"HOSTNAME": hostname,
},
)
l.envMtx.RUnlock()
if err != nil {
log.Error("error writing config", "err", err)
return err
}
config.Mounts = append(config.Mounts, bindMount(configPath, "/.containerconfig", false))
if job.Config.HostNetwork {
// allow host network jobs to configure the network
config.Capabilities = append(config.Capabilities, "CAP_NET_ADMIN")
} else {
ifaceName, err := netutils.GenerateIfaceName("veth", 4)
if err != nil {
return err
}
config.Hostname = hostname
config.Namespaces = append(config.Namespaces, configs.Namespace{Type: configs.NEWNET})
config.Networks = []*configs.Network{
{
Type: "loopback",
Address: "127.0.0.1/0",
Gateway: "localhost",
},
{
Type: "veth",
Name: "eth0",
Bridge: l.BridgeName,
Address: initConfig.IP,
Gateway: initConfig.Gateway,
Mtu: 1500,
HostInterfaceName: ifaceName,
},
}
}
if spec, ok := job.Resources[resource.TypeMemory]; ok && spec.Limit != nil {
config.Cgroups.Resources.Memory = *spec.Limit
}
if spec, ok := job.Resources[resource.TypeCPU]; ok && spec.Limit != nil {
config.Cgroups.Resources.CpuShares = milliCPUToShares(*spec.Limit)
}
c, err := l.factory.Create(job.ID, config)
if err != nil {
return err
}
process := &libcontainer.Process{
Args: []string{"/.containerinit", job.ID},
User: "******",
}
if err := c.Run(process); err != nil {
c.Destroy()
return err
}
go process.Wait()
container.container = c
go container.watch(nil, nil)
log.Info("job started")
return nil
}
func (e *endpoints) create(eid string, ifInfo *driverapi.EndpointInterface, niConfig networkConfig) (err error) {
ep := endpoint{}
// Generate a name for what will be the host side pipe interface
hostIfName, err := netutils.GenerateIfaceName(vethPrefix, vethLen)
if err != nil {
return err
}
// Generate a name for what will be the sandbox side pipe interface
containerIfName, err := netutils.GenerateIfaceName(vethPrefix, vethLen)
if err != nil {
return err
}
// Generate and add the interface pipe host <-> sandbox
veth := &netlink.Veth{
LinkAttrs: netlink.LinkAttrs{Name: hostIfName, TxQLen: 0},
PeerName: containerIfName,
}
if err = netlink.LinkAdd(veth); err != nil {
return types.InternalErrorf("failed to add the host (%s) <=> sandbox (%s) pair interfaces: %v", hostIfName, containerIfName, err)
}
// Get the host side pipe interface handler
host, err := netlink.LinkByName(hostIfName)
if err != nil {
return types.InternalErrorf("failed to find host side interface %s: %v", hostIfName, err)
}
defer func() {
if err != nil {
netlink.LinkDel(host)
}
}()
// Get the sandbox side pipe interface handler
sbox, err := netlink.LinkByName(containerIfName)
if err != nil {
return types.InternalErrorf("failed to find sandbox side interface %s: %v", containerIfName, err)
}
defer func() {
if err != nil {
netlink.LinkDel(sbox)
}
}()
// Add bridge inherited attributes to pipe interfaces
if niConfig.Mtu != 0 {
err = netlink.LinkSetMTU(host, niConfig.Mtu)
if err != nil {
return types.InternalErrorf("failed to set MTU on host interface %s: %v", hostIfName, err)
}
err = netlink.LinkSetMTU(sbox, niConfig.Mtu)
if err != nil {
return types.InternalErrorf("failed to set MTU on sandbox interface %s: %v", containerIfName, err)
}
}
// Attach host side pipe interface into the bridge
br, err := netlink.LinkByName(niConfig.BridgeName)
if err != nil {
return types.InternalErrorf("failed to find bridge by name %s: %v", niConfig.BridgeName, err)
}
if err = netlink.LinkSetMaster(host, br.(*netlink.Bridge)); err != nil {
return fmt.Errorf("adding interface %s to bridge %s failed: %v", hostIfName, niConfig.BridgeName, err)
}
// Create the sandbox side pipe interface
ep.ifname = containerIfName
ep.addr, _, err = net.ParseCIDR(ifInfo.Address)
if err != nil {
return fmt.Errorf("ipv4 adress unparseable")
}
/*
_, ep.addrv6, err = net.ParseCIDR(ifInfo.AddressIPv6)
if err != nil {
return fmt.Errorf("ipv6 adress unparseable")
}
*/
if ifInfo.MacAddress != "" {
ep.mac, err = net.ParseMAC(ifInfo.MacAddress)
if err != nil {
return fmt.Errorf("mac adress unparseable")
}
// Down the interface before configuring mac address.
if err = netlink.LinkSetDown(sbox); err != nil {
return fmt.Errorf("could not set link down for container interface %s: %v", containerIfName, err)
}
err = netlink.LinkSetHardwareAddr(sbox, ep.mac)
if err != nil {
return fmt.Errorf("could not set mac address for container interface %s: %v", containerIfName, err)
}
if err = netlink.LinkSetUp(sbox); err != nil {
return fmt.Errorf("could not set link up for container interface %s: %v", containerIfName, err)
}
} else {
// Get existing mac address from interface
ep.mac = sbox.Attrs().HardwareAddr
}
// Up the host interface after finishing all netlink configuration
if err = netlink.LinkSetUp(host); err != nil {
return fmt.Errorf("could not set link up for host interface %s: %v", hostIfName, err)
}
if ep.addrv6 == nil && niConfig.EnableIPv6 {
return fmt.Errorf("IPV6 is not supported. Go and code it yourself.")
}
e.add(eid, ep)
Log.Debugf("ep data at join: ip: %v, mac: %v", ep.addr, ep.mac)
broadcastChange(br, ep)
return nil
}
func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, epOptions map[string]interface{}) error {
defer osl.InitOSContext()()
if ifInfo == nil {
return errors.New("invalid interface info passed")
}
// Get the network handler and make sure it exists
d.Lock()
n, ok := d.networks[nid]
dconfig := d.config
d.Unlock()
if !ok {
return types.NotFoundErrorf("network %s does not exist", nid)
}
if n == nil {
return driverapi.ErrNoNetwork(nid)
}
// Sanity check
n.Lock()
if n.id != nid {
n.Unlock()
return InvalidNetworkIDError(nid)
}
n.Unlock()
// Check if endpoint id is good and retrieve correspondent endpoint
ep, err := n.getEndpoint(eid)
if err != nil {
return err
}
// Endpoint with that id exists either on desired or other sandbox
if ep != nil {
return driverapi.ErrEndpointExists(eid)
}
// Try to convert the options to endpoint configuration
epConfig, err := parseEndpointOptions(epOptions)
if err != nil {
return err
}
// Create and add the endpoint
n.Lock()
endpoint := &bridgeEndpoint{id: eid, config: epConfig}
n.endpoints[eid] = endpoint
n.Unlock()
// On failure make sure to remove the endpoint
defer func() {
if err != nil {
n.Lock()
delete(n.endpoints, eid)
n.Unlock()
}
}()
// Generate a name for what will be the host side pipe interface
hostIfName, err := netutils.GenerateIfaceName(vethPrefix, vethLen)
if err != nil {
return err
}
// Generate a name for what will be the sandbox side pipe interface
containerIfName, err := netutils.GenerateIfaceName(vethPrefix, vethLen)
if err != nil {
return err
}
// Generate and add the interface pipe host <-> sandbox
veth := &netlink.Veth{
LinkAttrs: netlink.LinkAttrs{Name: hostIfName, TxQLen: 0},
PeerName: containerIfName}
if err = netlink.LinkAdd(veth); err != nil {
return types.InternalErrorf("failed to add the host (%s) <=> sandbox (%s) pair interfaces: %v", hostIfName, containerIfName, err)
}
// Get the host side pipe interface handler
host, err := netlink.LinkByName(hostIfName)
if err != nil {
return types.InternalErrorf("failed to find host side interface %s: %v", hostIfName, err)
}
defer func() {
if err != nil {
netlink.LinkDel(host)
}
}()
// Get the sandbox side pipe interface handler
sbox, err := netlink.LinkByName(containerIfName)
if err != nil {
return types.InternalErrorf("failed to find sandbox side interface %s: %v", containerIfName, err)
}
defer func() {
if err != nil {
netlink.LinkDel(sbox)
}
}()
n.Lock()
config := n.config
n.Unlock()
// Add bridge inherited attributes to pipe interfaces
if config.Mtu != 0 {
err = netlink.LinkSetMTU(host, config.Mtu)
if err != nil {
return types.InternalErrorf("failed to set MTU on host interface %s: %v", hostIfName, err)
}
err = netlink.LinkSetMTU(sbox, config.Mtu)
if err != nil {
return types.InternalErrorf("failed to set MTU on sandbox interface %s: %v", containerIfName, err)
}
}
// Attach host side pipe interface into the bridge
if err = addToBridge(hostIfName, config.BridgeName); err != nil {
return fmt.Errorf("adding interface %s to bridge %s failed: %v", hostIfName, config.BridgeName, err)
}
if !dconfig.EnableUserlandProxy {
err = setHairpinMode(host, true)
if err != nil {
return err
}
}
// Create the sandbox side pipe interface
endpoint.srcName = containerIfName
endpoint.macAddress = ifInfo.MacAddress()
endpoint.addr = ifInfo.Address()
endpoint.addrv6 = ifInfo.AddressIPv6()
// Down the interface before configuring mac address.
if err = netlink.LinkSetDown(sbox); err != nil {
return fmt.Errorf("could not set link down for container interface %s: %v", containerIfName, err)
}
// Set the sbox's MAC. If specified, use the one configured by user, otherwise generate one based on IP.
if endpoint.macAddress == nil {
endpoint.macAddress = electMacAddress(epConfig, endpoint.addr.IP)
if err := ifInfo.SetMacAddress(endpoint.macAddress); err != nil {
return err
}
}
err = netlink.LinkSetHardwareAddr(sbox, endpoint.macAddress)
if err != nil {
return fmt.Errorf("could not set mac address for container interface %s: %v", containerIfName, err)
}
// Up the host interface after finishing all netlink configuration
if err = netlink.LinkSetUp(host); err != nil {
return fmt.Errorf("could not set link up for host interface %s: %v", hostIfName, err)
}
if endpoint.addrv6 == nil && config.EnableIPv6 {
var ip6 net.IP
network := n.bridge.bridgeIPv6
if config.AddressIPv6 != nil {
network = config.AddressIPv6
}
ones, _ := network.Mask.Size()
if ones > 80 {
err = types.ForbiddenErrorf("Cannot self generate an IPv6 address on network %v: At least 48 host bits are needed.", network)
return err
}
ip6 = make(net.IP, len(network.IP))
copy(ip6, network.IP)
for i, h := range endpoint.macAddress {
ip6[i+10] = h
}
endpoint.addrv6 = &net.IPNet{IP: ip6, Mask: network.Mask}
if err := ifInfo.SetIPAddress(endpoint.addrv6); err != nil {
return err
}
}
// Add a neighbor proxy if using NDP proxying
if config.NDPProxyInterface != "" && config.EnableIPv6 {
link, err := netlink.LinkByName(config.NDPProxyInterface)
if err != nil {
return err
}
neighbor := netlink.Neigh{
LinkIndex: link.Attrs().Index,
Family: netlink.FAMILY_V6,
State: netlink.NUD_PERMANENT,
Type: netlink.NDA_UNSPEC,
Flags: netlink.NTF_PROXY,
IP: endpoint.addrv6.IP,
HardwareAddr: endpoint.macAddress,
}
if err := netlink.NeighAdd(&neighbor); err != nil {
logrus.Warnf("could not add the neighbor proxy: %v", err)
return err
}
if endpoint.config != nil {
for _, port := range endpoint.config.ExposedPorts {
insert := []string{
string(iptables.Insert),
DockerChain,
"-p", port.Proto.String(),
"-d", endpoint.addrv6.String(),
"--dport", strconv.Itoa(int(port.Port)),
"-j", "ACCEPT",
}
iptables.Raw(iptables.IP6Tables, insert...)
}
}
}
// Program any required port mapping and store them in the endpoint
endpoint.portMapping, err = n.allocatePorts(epConfig, endpoint, config.DefaultBindingIP, d.config.EnableUserlandProxy)
if err != nil {
return err
}
return nil
}