// GC enters the pod by fork/exec()ing the stage1's /gc similar to /init.
// /gc can expect to have its CWD set to the pod root.
// stage1Path is the path of the stage1 rootfs
func GC(pdir string, uuid *types.UUID, stage1Path string, debug bool) error {
err := unregisterPod(pdir, uuid)
if err != nil {
// Probably not worth abandoning the rest
log.Printf("Warning: could not unregister pod with metadata service: %v", err)
}
ep, err := getStage1Entrypoint(pdir, gcEntrypoint)
if err != nil {
return fmt.Errorf("error determining gc entrypoint: %v", err)
}
args := []string{filepath.Join(stage1Path, ep)}
if debug {
args = append(args, "--debug")
}
args = append(args, uuid.String())
c := exec.Cmd{
Path: args[0],
Args: args,
Stderr: os.Stderr,
Dir: pdir,
}
return c.Run()
}
// registerPod registers pod with metadata service.
// Returns authentication token to be passed in the URL
func registerPod(root string, uuid *types.UUID, apps schema.AppList) (token string, rerr error) {
u := uuid.String()
var err error
token, err = generateMDSToken()
if err != nil {
rerr = fmt.Errorf("failed to generate MDS token: %v", err)
return
}
pmfPath := common.PodManifestPath(root)
pmf, err := os.Open(pmfPath)
if err != nil {
rerr = fmt.Errorf("failed to open runtime manifest (%v): %v", pmfPath, err)
return
}
pth := fmt.Sprintf("/pods/%v?token=%v", u, token)
err = httpRequest("PUT", pth, pmf)
pmf.Close()
if err != nil {
rerr = fmt.Errorf("failed to register pod with metadata svc: %v", err)
return
}
defer func() {
if rerr != nil {
unregisterPod(root, uuid)
}
}()
rf, err := os.Create(filepath.Join(root, mdsRegisteredFile))
if err != nil {
rerr = fmt.Errorf("failed to create mds-register file: %v", err)
return
}
rf.Close()
for _, app := range apps {
ampath := common.ImageManifestPath(root, app.Name)
amf, err := os.Open(ampath)
if err != nil {
rerr = fmt.Errorf("failed reading app manifest %q: %v", ampath, err)
return
}
err = registerApp(u, app.Name.String(), amf)
amf.Close()
if err != nil {
rerr = fmt.Errorf("failed to register app with metadata svc: %v", err)
return
}
}
return
}
// unregisterPod unregisters pod with the metadata service.
func unregisterPod(root string, uuid *types.UUID) error {
_, err := os.Stat(filepath.Join(root, mdsRegisteredFile))
switch {
case err == nil:
pth := path.Join("/pods", uuid.String())
return httpRequest("DELETE", pth, nil)
case os.IsNotExist(err):
return nil
default:
return err
}
}
// setupTapDevice creates persistent tap devices
// and returns a newly created netlink.Link structure
func setupTapDevice(podID types.UUID) (netlink.Link, error) {
// network device names are limited to 16 characters
// the suffix %d will be replaced by the kernel with a suitable number
nameTemplate := fmt.Sprintf("rkt-%s-tap%%d", podID.String()[0:4])
ifName, err := tuntap.CreatePersistentIface(nameTemplate, tuntap.Tap)
if err != nil {
return nil, fmt.Errorf("tuntap persist %v", err)
}
link, err := netlink.LinkByName(ifName)
if err != nil {
return nil, fmt.Errorf("cannot find link %q: %v", ifName, err)
}
err = netlink.LinkSetUp(link)
if err != nil {
return nil, fmt.Errorf("cannot set link up %q: %v", ifName, err)
}
return link, nil
}
// GC enters the pod by fork/exec()ing the stage1's /gc similar to /init.
// /gc can expect to have its CWD set to the pod root.
// stage1Path is the path of the stage1 rootfs
func GC(pdir string, uuid *types.UUID, stage1Path string, debug bool) error {
ep, err := getStage1Entrypoint(pdir, gcEntrypoint)
if err != nil {
return fmt.Errorf("error determining gc entrypoint: %v", err)
}
args := []string{filepath.Join(stage1Path, ep)}
if debug {
args = append(args, "--debug")
}
args = append(args, uuid.String())
c := exec.Cmd{
Path: args[0],
Args: args,
Stderr: os.Stderr,
Dir: pdir,
}
return c.Run()
}
// setupTapDevice creates persistent macvtap device
// and returns a newly created netlink.Link structure
func setupMacVTapDevice(podID types.UUID, config MacVTapNetConf) (netlink.Link, error) {
master, err := netlink.LinkByName(config.Master)
if err != nil {
return nil, fmt.Errorf("Cannot find master device '%v': %v", config.Master, err)
}
var mode netlink.MacvlanMode
switch config.Mode {
// if not set - defaults to bridge mode as in:
// https://github.com/coreos/rkt/blob/master/Documentation/networking.md#macvlan
case "", "bridge":
mode = netlink.MACVLAN_MODE_BRIDGE
case "private":
mode = netlink.MACVLAN_MODE_PRIVATE
case "vepa":
mode = netlink.MACVLAN_MODE_VEPA
case "passthru":
mode = netlink.MACVLAN_MODE_PASSTHRU
default:
return nil, fmt.Errorf("Unsupported macvtap mode: %v", config.Mode)
}
mtu := master.Attrs().MTU
if config.MTU != 0 {
mtu = config.MTU
}
nameTemplate := fmt.Sprintf("rkt-%s-vtap%%d", podID.String()[0:4])
link := &netlink.Macvtap{
Macvlan: netlink.Macvlan{
LinkAttrs: netlink.LinkAttrs{
Name: nameTemplate,
MTU: mtu,
ParentIndex: master.Attrs().Index,
},
Mode: mode,
},
}
if err := netlink.LinkAdd(link); err != nil {
return nil, fmt.Errorf("Cannot create macvtap interface: %v", err)
}
return link, nil
}
// kvmSetup prepare new Networking to be used in kvm environment based on tuntap pair interfaces
// to allow communication with virtual machine created by lkvm tool
// right now it only supports default "ptp" network type (other types ends with error)
func kvmSetup(podRoot string, podID types.UUID, fps []ForwardedPort, privateNetList common.PrivateNetList, localConfig string) (*Networking, error) {
network := Networking{
podEnv: podEnv{
podRoot: podRoot,
podID: podID,
netsLoadList: privateNetList,
localConfig: localConfig,
},
}
var e error
network.nets, e = network.loadNets()
if e != nil {
return nil, fmt.Errorf("error loading network definitions: %v", e)
}
for _, n := range network.nets {
switch n.conf.Type {
case "ptp":
link, err := setupTapDevice()
if err != nil {
return nil, err
}
ifName := link.Attrs().Name
n.runtime.IfName = ifName
err = kvmSetupNetAddressing(&network, n, ifName)
if err != nil {
return nil, err
}
// add address to host tap device
err = netlink.AddrAdd(
link,
&netlink.Addr{
IPNet: &net.IPNet{
IP: n.runtime.HostIP,
Mask: net.IPMask(n.runtime.Mask),
},
Label: ifName,
})
if err != nil {
return nil, fmt.Errorf("cannot add address to host tap device %q: %v", ifName, err)
}
if n.conf.IPMasq {
h := sha512.Sum512([]byte(podID.String()))
chain := fmt.Sprintf("CNI-%s-%x", n.conf.Name, h[:8])
if err = ip.SetupIPMasq(&net.IPNet{
IP: n.runtime.IP,
Mask: net.IPMask(n.runtime.Mask),
}, chain); err != nil {
return nil, err
}
}
default:
return nil, fmt.Errorf("network %q have unsupported type: %q", n.conf.Name, n.conf.Type)
}
}
err := network.forwardPorts(fps, network.GetDefaultIP())
if err != nil {
return nil, err
}
return &network, nil
}
// unregisterPod unregisters pod with the metadata service.
func unregisterPod(uuid *types.UUID) error {
pth := path.Join("/pods", uuid.String())
return httpRequest("DELETE", pth, nil)
}
// kvmSetup prepare new Networking to be used in kvm environment based on tuntap pair interfaces
// to allow communication with virtual machine created by lkvm tool
func kvmSetup(podRoot string, podID types.UUID, fps []ForwardedPort, netList common.NetList, localConfig string) (*Networking, error) {
network := Networking{
podEnv: podEnv{
podRoot: podRoot,
podID: podID,
netsLoadList: netList,
localConfig: localConfig,
},
}
var e error
network.nets, e = network.loadNets()
if e != nil {
return nil, fmt.Errorf("error loading network definitions: %v", e)
}
for i, n := range network.nets {
if n.conf.Type == "flannel" {
if err := kvmTransformFlannelNetwork(&n); err != nil {
return nil, fmt.Errorf("cannot transform flannel network into basic network: %v", err)
}
}
switch n.conf.Type {
case "ptp":
link, err := setupTapDevice(podID)
if err != nil {
return nil, err
}
ifName := link.Attrs().Name
n.runtime.IfName = ifName
err = kvmSetupNetAddressing(&network, n, ifName)
if err != nil {
return nil, err
}
// add address to host tap device
err = ensureHasAddr(
link,
&net.IPNet{
IP: n.runtime.IP4.Gateway,
Mask: net.IPMask(n.runtime.Mask),
},
)
if err != nil {
return nil, fmt.Errorf("cannot add address to host tap device %q: %v", ifName, err)
}
if err := removeAllRoutesOnLink(link); err != nil {
return nil, fmt.Errorf("cannot remove route on host tap device %q: %v", ifName, err)
}
if err := addRoute(link, n.runtime.IP); err != nil {
return nil, fmt.Errorf("cannot add on host direct route to pod: %v", err)
}
case "bridge":
config := BridgeNetConf{
NetConf: NetConf{
MTU: defaultMTU,
},
BrName: defaultBrName,
}
if err := json.Unmarshal(n.confBytes, &config); err != nil {
return nil, fmt.Errorf("error parsing %q result: %v", n.conf.Name, err)
}
br, err := ensureBridgeIsUp(config.BrName, config.MTU)
if err != nil {
return nil, fmt.Errorf("error in time of bridge setup: %v", err)
}
link, err := setupTapDevice(podID)
if err != nil {
return nil, fmt.Errorf("can not setup tap device: %v", err)
}
err = netlink.LinkSetMaster(link, br)
if err != nil {
rErr := tuntap.RemovePersistentIface(n.runtime.IfName, tuntap.Tap)
if rErr != nil {
log.Printf("Warning: could not cleanup tap interface: %v", rErr)
}
return nil, fmt.Errorf("can not add tap interface to bridge: %v", err)
}
ifName := link.Attrs().Name
n.runtime.IfName = ifName
err = kvmSetupNetAddressing(&network, n, ifName)
if err != nil {
return nil, err
}
if config.IsGw {
err = ensureHasAddr(
br,
&net.IPNet{
IP: n.runtime.IP4.Gateway,
Mask: net.IPMask(n.runtime.Mask),
},
)
if err != nil {
return nil, fmt.Errorf("cannot add address to host bridge device %q: %v", br.Name, err)
}
}
case "macvlan":
config := MacVTapNetConf{}
if err := json.Unmarshal(n.confBytes, &config); err != nil {
return nil, fmt.Errorf("error parsing %q result: %v", n.conf.Name, err)
}
link, err := setupMacVTapDevice(podID, config)
if err != nil {
return nil, err
}
ifName := link.Attrs().Name
n.runtime.IfName = ifName
err = kvmSetupNetAddressing(&network, n, ifName)
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("network %q have unsupported type: %q", n.conf.Name, n.conf.Type)
}
if n.conf.IPMasq {
chain := getChainName(podID.String(), n.conf.Name)
if err := ip.SetupIPMasq(&net.IPNet{
IP: n.runtime.IP,
Mask: net.IPMask(n.runtime.Mask),
}, chain); err != nil {
return nil, err
}
}
network.nets[i] = n
}
err := network.forwardPorts(fps, network.GetDefaultIP())
if err != nil {
return nil, err
}
return &network, nil
}
func writeUUIDToFile(uuid *types.UUID, path string) error {
return ioutil.WriteFile(path, []byte(uuid.String()), 0644)
}
