// sets IP4 addr on link removing any existing ones first
func setAddr4(link *netlink.Vxlan, ipn *net.IPNet) error {
addrs, err := netlink.AddrList(link, syscall.AF_INET)
if err != nil {
return err
}
addr := netlink.Addr{ipn, ""}
existing := false
for _, old := range addrs {
if old.IPNet.String() == addr.IPNet.String() {
existing = true
continue
}
if err = netlink.AddrDel(link, &old); err != nil {
return fmt.Errorf("failed to delete IPv4 addr %s from %s", old.String(), link.Attrs().Name)
}
}
if !existing {
if err = netlink.AddrAdd(link, &addr); err != nil {
return fmt.Errorf("failed to add IP address %s to %s: %s", ipn.String(), link.Attrs().Name, err)
}
}
return nil
}
func (mgr *filterChain) Setup(instanceChain, bridgeName string, ip net.IP, network *net.IPNet) error {
commands := []*exec.Cmd{
// Create filter instance chain
exec.Command("iptables", "--wait", "-N", instanceChain),
// Allow intra-subnet traffic (Linux ethernet bridging goes through ip stack)
exec.Command("iptables", "--wait", "-A", instanceChain, "-s", network.String(), "-d", network.String(), "-j", "ACCEPT"),
// Otherwise, use the default filter chain
exec.Command("iptables", "--wait", "-A", instanceChain, "--goto", mgr.cfg.DefaultChain),
// Bind filter instance chain to filter forward chain
exec.Command("iptables", "--wait", "-I", mgr.cfg.ForwardChain, "2", "--in-interface", bridgeName, "--source", ip.String(), "--goto", instanceChain),
}
for _, cmd := range commands {
buffer := &bytes.Buffer{}
cmd.Stderr = buffer
logger := mgr.logger.Session("setup", lager.Data{"cmd": cmd})
logger.Debug("starting")
if err := mgr.runner.Run(cmd); err != nil {
stderr, _ := ioutil.ReadAll(buffer)
logger.Error("failed", err, lager.Data{"stderr": string(stderr)})
return fmt.Errorf("iptables_manager: filter: %s", err)
}
logger.Debug("ended")
}
return nil
}
func generateCMD(dst net.IPNet, gw net.IP, is_delete bool) []string {
var netmask_str string = net.IP(dst.Mask).String()
var ipaddr_str string = dst.IP.String()
var gateway_str string = gw.String()
var network_str string = dst.String()
switch runtime.GOOS {
case "darwin":
if is_delete {
return []string{"route", "delete", "-net",
ipaddr_str, gateway_str, netmask_str}
} else {
return []string{"route", "add", "-net",
ipaddr_str, gateway_str, netmask_str}
}
case "linux":
if is_delete {
return []string{"ip", "route", "del", network_str}
} else {
return []string{"ip", "route", "add", network_str, "via", gateway_str}
}
default:
return nil
}
return nil
}
func ensureBridgeAddr(br *netlink.Bridge, ipn *net.IPNet, forceAddress bool) error {
addrs, err := netlink.AddrList(br, syscall.AF_INET)
if err != nil && err != syscall.ENOENT {
return fmt.Errorf("could not get list of IP addresses: %v", err)
}
// if there're no addresses on the bridge, it's ok -- we'll add one
if len(addrs) > 0 {
ipnStr := ipn.String()
for _, a := range addrs {
// string comp is actually easiest for doing IPNet comps
if a.IPNet.String() == ipnStr {
return nil
}
// If forceAddress is set to true then reconfigure IP address otherwise throw error
if forceAddress {
if err = deleteBridgeAddr(br, a.IPNet); err != nil {
return err
}
} else {
return fmt.Errorf("%q already has an IP address different from %v", br.Name, ipn.String())
}
}
}
addr := &netlink.Addr{IPNet: ipn, Label: ""}
if err := netlink.AddrAdd(br, addr); err != nil {
return fmt.Errorf("could not add IP address to %q: %v", br.Name, err)
}
return nil
}
func (a *Allocator) insertBitMask(key SubnetKey, pool *net.IPNet) error {
log.Debugf("Inserting bitmask (%s, %s)", key.String(), pool.String())
store := a.getStore(key.AddressSpace)
if store == nil {
return fmt.Errorf("could not find store for address space %s while inserting bit mask", key.AddressSpace)
}
ipVer := getAddressVersion(pool.IP)
ones, bits := pool.Mask.Size()
numAddresses := uint32(1 << uint(bits-ones))
if ipVer == v4 {
// Do not let broadcast address be reserved
numAddresses--
}
// Generate the new address masks. AddressMask content may come from datastore
h, err := bitseq.NewHandle(dsDataKey, store, key.String(), numAddresses)
if err != nil {
return err
}
if ipVer == v4 {
// Do not let network identifier address be reserved
h.Set(0)
}
a.Lock()
a.addresses[key] = h
a.Unlock()
return nil
}
func (a *Allocator) parsePoolRequest(addressSpace, pool, subPool string, v6 bool) (*SubnetKey, *net.IPNet, *AddressRange, error) {
var (
nw *net.IPNet
ipr *AddressRange
err error
)
if addressSpace == "" {
return nil, nil, nil, ipamapi.ErrInvalidAddressSpace
}
if pool == "" && subPool != "" {
return nil, nil, nil, ipamapi.ErrInvalidSubPool
}
if pool == "" {
return nil, nil, nil, nil
}
if _, nw, err = net.ParseCIDR(pool); err != nil {
return nil, nil, nil, ipamapi.ErrInvalidPool
}
if subPool != "" {
if ipr, err = getAddressRange(subPool, nw); err != nil {
return nil, nil, nil, err
}
}
return &SubnetKey{AddressSpace: addressSpace, Subnet: nw.String(), ChildSubnet: subPool}, nw, ipr, nil
}
// return an available ip if one is currently available. If not,
// return the next available ip for the nextwork
func getNextIp(address *net.IPNet) (*net.IP, error) {
var (
ownIP = ipToInt(&address.IP)
allocated = allocatedIPs[address.String()]
first, _ = networkdriver.NetworkRange(address)
base = ipToInt(&first)
size = int(networkdriver.NetworkSize(address.Mask))
max = int32(size - 2) // size -1 for the broadcast address, -1 for the gateway address
pos = atomic.LoadInt32(&allocated.last)
)
var (
firstNetIP = address.IP.To4().Mask(address.Mask)
firstAsInt = ipToInt(&firstNetIP) + 1
)
for i := int32(0); i < max; i++ {
pos = pos%max + 1
next := int32(base + pos)
if next == ownIP || next == firstAsInt {
continue
}
if !allocated.Exists(int(pos)) {
ip := intToIP(next)
allocated.Push(int(pos))
atomic.StoreInt32(&allocated.last, pos)
return ip, nil
}
}
return nil, ErrNoAvailableIPs
}
func createCBR0(wantCIDR *net.IPNet) error {
// recreate cbr0 with wantCIDR
if err := exec.Command("brctl", "addbr", "cbr0").Run(); err != nil {
glog.Error(err)
return err
}
if err := exec.Command("ip", "addr", "add", wantCIDR.String(), "dev", "cbr0").Run(); err != nil {
glog.Error(err)
return err
}
if err := exec.Command("ip", "link", "set", "dev", "cbr0", "mtu", "1460", "up").Run(); err != nil {
glog.Error(err)
return err
}
// restart docker
// For now just log the error. The containerRuntime check will catch docker failures.
// TODO (dawnchen) figure out what we should do for rkt here.
if util.UsingSystemdInitSystem() {
if err := exec.Command("systemctl", "restart", "docker").Run(); err != nil {
glog.Error(err)
}
} else {
if err := exec.Command("service", "docker", "restart").Run(); err != nil {
glog.Error(err)
}
}
glog.V(2).Info("Recreated cbr0 and restarted docker")
return nil
}
// Initialize the repo to be used to announce/write config.
// A seperate repo is initialized to read incoming announcements
func initConfigWrite(networkCidr *net.IPNet, hostIface, gitRepoURL string) {
var err error
if !pathExists(EndpointPushSubDir) {
log.Debugf("[ %s ] dir not found, creating it..", EndpointPushSubDir)
if err = CreatePaths(EndpointPushSubDir); err != nil {
log.Fatalf("Could not create the directory [ %s ]: %s", EndpointPushSubDir, err)
} else {
log.Warnf("Succesfully created the config path [ %s ]", EndpointPushSubDir)
}
}
// Create the cache subdirectories
time.Sleep(1 * time.Second)
localEndpointIP, _ := getIfaceAddrStr(hostIface)
// Fun Go fact: using a + with sprintf is faster then %s since it uses reflection
endpointFile := fmt.Sprintf(localEndpointIP + dotjson)
log.Debugf("The endpoint file name is [ %s ] ", endpointFile)
log.Debugf("Anouncing this endpoint using the source [ %s ] and advertsing network [ %s ] to datastore file [ %s ]", networkCidr, localEndpointIP, endpointFile)
endpointConfig := &LocalEndpoint{
Endpoint: localEndpointIP,
Network: networkCidr.String(),
Meta: "",
}
var configAnnounce []LocalEndpoint
configAnnounce = append(configAnnounce, *endpointConfig)
marshallConfig(configAnnounce, configFormat, endpointFile)
if log.GetLevel().String() == "debug" {
printPretty(configAnnounce, "json")
}
// Parse the repo name
defer gitPushConfig()
}
func (master *OsdnMaster) SubnetStartMaster(clusterNetwork *net.IPNet, hostSubnetLength uint32) error {
subrange := make([]string, 0)
subnets, err := master.osClient.HostSubnets().List(kapi.ListOptions{})
if err != nil {
log.Errorf("Error in initializing/fetching subnets: %v", err)
return err
}
for _, sub := range subnets.Items {
subrange = append(subrange, sub.Subnet)
if err = master.networkInfo.validateNodeIP(sub.HostIP); err != nil {
// Don't error out; just warn so the error can be corrected with 'oc'
log.Errorf("Failed to validate HostSubnet %s: %v", hostSubnetToString(&sub), err)
} else {
log.Infof("Found existing HostSubnet %s", hostSubnetToString(&sub))
}
}
master.subnetAllocator, err = netutils.NewSubnetAllocator(clusterNetwork.String(), hostSubnetLength, subrange)
if err != nil {
return err
}
go utilwait.Forever(master.watchNodes, 0)
go utilwait.Forever(master.watchSubnets, 0)
return nil
}
func checkAddress(address *net.IPNet) {
key := address.String()
if _, exists := allocatedIPs[key]; !exists {
allocatedIPs[key] = &iPSet{}
availableIPS[key] = &iPSet{}
}
}
func (oc *OsdnController) SubnetStartMaster(clusterNetwork *net.IPNet, hostSubnetLength uint) error {
subrange := make([]string, 0)
subnets, err := oc.Registry.GetSubnets()
if err != nil {
log.Errorf("Error in initializing/fetching subnets: %v", err)
return err
}
for _, sub := range subnets {
subrange = append(subrange, sub.Subnet)
if err := oc.validateNode(sub.HostIP); err != nil {
// Don't error out; just warn so the error can be corrected with 'oc'
log.Errorf("Failed to validate HostSubnet %s: %v", err)
} else {
log.Infof("Found existing HostSubnet %s", HostSubnetToString(&sub))
}
}
oc.subnetAllocator, err = netutils.NewSubnetAllocator(clusterNetwork.String(), hostSubnetLength, subrange)
if err != nil {
return err
}
go utilwait.Forever(oc.watchNodes, 0)
return nil
}
func generateRandomNetwork(address *net.IPNet) string {
tick := float64(time.Now().UnixNano() / 1000000)
ones, bits := address.Mask.Size()
zeros := bits - ones
uniqIPsAmount := math.Pow(2.0, float64(zeros))
rawIP := math.Mod(tick, uniqIPsAmount)
remainder := rawIP
remainder, octet4 := math.Modf(remainder / 255.0)
remainder, octet3 := math.Modf(remainder / 255.0)
remainder, octet2 := math.Modf(remainder / 255.0)
base := address.IP
address.IP = net.IPv4(
byte(remainder)|base[0],
byte(octet2*255)|base[1],
byte(octet3*255)|base[2],
byte(octet4*255)|base[3],
)
address.IP.Mask(address.Mask)
return address.String()
}
func addRedirectRules(path string, eIP *net.IPNet, ingressPorts []*PortConfig) error {
var ingressPortsFile string
if len(ingressPorts) != 0 {
var err error
ingressPortsFile, err = writePortsToFile(ingressPorts)
if err != nil {
return err
}
defer os.Remove(ingressPortsFile)
}
cmd := &exec.Cmd{
Path: reexec.Self(),
Args: append([]string{"redirecter"}, path, eIP.String(), ingressPortsFile),
Stdout: os.Stdout,
Stderr: os.Stderr,
}
if err := cmd.Run(); err != nil {
return fmt.Errorf("reexec failed: %v", err)
}
return nil
}
func (this pool) NextAddr(args ...interface{}) string {
var addr, netmask string
var ipnet *net.IPNet
switch len(args) {
case 2:
addr = args[0].(string)
netmask = args[1].(string)
ipnet = &net.IPNet{
IP: net.ParseIP(addr),
Mask: net.IPMask(net.ParseIP(netmask)),
}
case 0:
for k, _ := range this.cache {
ipnet = this.cache[k]
break
}
default:
return ""
}
ip, _, _ := net.ParseCIDR(this.NextCidr(ipnet.String()))
return ip.String()
}
func setAddress(ifaceAddr *string, address *net.IPNet) error {
if *ifaceAddr != "" {
return types.ForbiddenErrorf("endpoint interface IP present (%s). Cannot be modified with (%s).", *ifaceAddr, address)
}
*ifaceAddr = address.String()
return nil
}
func checkAddress(address *net.IPNet) {
key := address.String()
if _, exists := allocatedIPs[key]; !exists {
allocatedIPs[key] = collections.NewOrderedIntSet()
availableIPS[key] = collections.NewOrderedIntSet()
}
}
func GenerateNetworkInterfaceUnits(unitsPath string, netDescriptions []netDescriber) error {
for i, netDescription := range netDescriptions {
ifName := fmt.Sprintf(networking.IfNamePattern, i)
netAddress := net.IPNet{
IP: netDescription.GuestIP(),
Mask: net.IPMask(netDescription.Mask()),
}
address := netAddress.String()
mac, err := generateMacAddress()
if err != nil {
return err
}
opts := []*unit.UnitOption{
unit.NewUnitOption("Unit", "Description", fmt.Sprintf("Network configuration for device: %v", ifName)),
unit.NewUnitOption("Unit", "DefaultDependencies", "false"),
unit.NewUnitOption("Service", "Type", "oneshot"),
unit.NewUnitOption("Service", "RemainAfterExit", "true"),
unit.NewUnitOption("Service", "ExecStartPre", downInterfaceCommand(ifName)),
unit.NewUnitOption("Service", "ExecStartPre", setMacCommand(ifName, mac.String())),
unit.NewUnitOption("Service", "ExecStartPre", upInterfaceCommand(ifName)),
unit.NewUnitOption("Service", "ExecStart", addAddressCommand(address, ifName)),
unit.NewUnitOption("Install", "RequiredBy", "default.target"),
}
for _, route := range netDescription.Routes() {
gw := route.GW
if gw == nil {
gw = netDescription.Gateway()
}
opts = append(
opts,
unit.NewUnitOption(
"Service",
"ExecStartPost",
addRouteCommand(route.Dst.String(), gw.String()),
),
)
}
unitName := fmt.Sprintf("interface-%s", ifName) + ".service"
unitBytes, err := ioutil.ReadAll(unit.Serialize(opts))
if err != nil {
return fmt.Errorf("failed to serialize network unit file to bytes %q: %v", unitName, err)
}
err = ioutil.WriteFile(filepath.Join(unitsPath, unitName), unitBytes, 0644)
if err != nil {
return fmt.Errorf("failed to create network unit file %q: %v", unitName, err)
}
log.Printf("network unit created: %q in %q (iface=%q, addr=%q)", unitName, unitsPath, ifName, address)
}
return nil
}
// Advertise the local namespace IP prefixes to the bgp neighbors
func OriginateBgpRoute(localPrefix *net.IPNet) error {
log.Infof("Adding this hosts container network [ %s ] into the BGP domain", localPrefix)
_, stderr, err := gobgp(bgpCmd, global, rib, bgpAdd, localPrefix.String(), addrFamily, ipv4)
if err != nil {
return errors.New(stderr.String())
}
return nil
}
func (test *testEndpoint) SetIPAddress(address *net.IPNet) error {
if address.IP == nil {
return types.BadRequestErrorf("tried to set nil IP address to endpoint interface")
}
test.address = address.String()
return nil
}
func setupIfce(t *testing.T, ipNet net.IPNet, dev string) {
if err := exec.Command("ip", "link", "set", dev, "up").Run(); err != nil {
t.Fatal(err)
}
if err := exec.Command("ip", "addr", "add", ipNet.String(), "dev", dev).Run(); err != nil {
t.Fatal(err)
}
}
// Pass an IPNet object to find the value match in leveldb and return
// both the matching object (or nil) and the error (or nil)
func (db *DB) GetEntry(cidr *net.IPNet) (*heimdall.Entry, error) {
if entry, err := db.Get([]byte(cidr.String()), nil); err == nil {
timestamp, _ := time.Parse(TIMEFORMAT, string(entry))
return &heimdall.Entry{cidr, timestamp}, nil
} else {
return nil, err
}
}
// ReleaseIP adds the provided ip back into the pool of
// available ips to be returned for use.
func ReleaseIP(network *net.IPNet, ip net.IP) error {
lock.Lock()
defer lock.Unlock()
if allocated, exists := allocatedIPs[network.String()]; exists {
delete(allocated.p, ip.String())
}
return nil
}
// AddVIPSubnet adds a VIP Subnet to a Seesaw Cluster.
func (c *Cluster) AddVIPSubnet(subnet *net.IPNet) error {
key := subnet.String()
if _, ok := c.VIPSubnets[key]; ok {
return fmt.Errorf("Cluster %q already contains VIP Subnet %q", c.Site, key)
}
c.VIPSubnets[key] = subnet
return nil
}
// ReleaseIP adds the provided ip back into the pool of
// available ips to be returned for use.
func ReleaseIP(network *net.IPNet, ip *net.IP) error {
lock.Lock()
defer lock.Unlock()
if allocated, exists := allocatedIPs[network.String()]; exists {
pos := getPosition(network, ip)
delete(allocated.p, pos)
}
return nil
}
// ReleaseIP adds the provided ip back into the pool of
// available ips to be returned for use.
func (a *IPAllocator) ReleaseIP(network *net.IPNet, ip net.IP) error {
a.mutex.Lock()
defer a.mutex.Unlock()
if allocated, exists := a.allocatedIPs[network.String()]; exists {
delete(allocated.p, ip.String())
}
return nil
}
// RemoveSubnet removes the subnet from the specified address space
func (a *Allocator) RemoveSubnet(addrSpace AddressSpace, subnet *net.IPNet) error {
if addrSpace == "" {
return ErrInvalidAddressSpace
}
if subnet == nil {
return ErrInvalidSubnet
}
retry:
// Look for the respective subnet configuration data
// Remove it along with the internal subnets
subKey := subnetKey{addrSpace, subnet.String(), ""}
a.Lock()
current, ok := a.subnets[subKey]
a.Unlock()
if !ok {
return ErrSubnetNotFound
}
// Remove config and sync to datastore
a.Lock()
delete(a.subnets, subKey)
a.Unlock()
err := a.writeToStore()
if err != nil {
if _, ok := err.(types.RetryError); !ok {
return types.InternalErrorf("subnet removal failed because of %s", err.Error())
}
// Update to latest
if erru := a.readFromStore(); erru != nil {
// Restore and bail out
a.Lock()
a.subnets[subKey] = current
a.Unlock()
return fmt.Errorf("failed to get updated subnets config from datastore (%v) after (%v)", erru, err)
}
goto retry
}
// Get the list of smaller internal subnets
subnetList, err := getInternalSubnets(subnet, a.internalHostSize)
if err != nil {
return err
}
for _, s := range subnetList {
sk := subnetKey{addrSpace, subKey.subnet, s.String()}
a.Lock()
if bm, ok := a.addresses[sk]; ok {
bm.Destroy()
}
delete(a.addresses, sk)
a.Unlock()
}
return nil
}
func (oc *OsdnController) SubnetStartMaster(clusterNetwork *net.IPNet, hostSubnetLength uint) error {
subrange := make([]string, 0)
subnets, _, err := oc.Registry.GetSubnets()
if err != nil {
log.Errorf("Error in initializing/fetching subnets: %v", err)
return err
}
for _, sub := range subnets {
subrange = append(subrange, sub.Subnet)
if err := oc.validateNode(sub.HostIP); err != nil {
// Don't error out; just warn so the error can be corrected with 'oc'
log.Errorf("Failed to validate HostSubnet %s: %v", err)
}
}
oc.subnetAllocator, err = netutils.NewSubnetAllocator(clusterNetwork.String(), hostSubnetLength, subrange)
if err != nil {
return err
}
getNodes := func(registry *Registry) (interface{}, string, error) {
return registry.GetNodes()
}
result, err := oc.watchAndGetResource("Node", watchNodes, getNodes)
if err != nil {
return err
}
// Make sure each node has a Subnet allocated
nodes := result.([]kapi.Node)
for _, node := range nodes {
nodeIP, err := GetNodeIP(&node)
if err != nil {
// Don't error out; just warn so the error can be corrected by admin
log.Errorf("Failed to get Node %s IP: %v", node.Name, err)
continue
}
err = oc.validateNode(nodeIP)
if err != nil {
// Don't error out; just warn so the error can be corrected by admin
log.Errorf("Failed to validate Node %s: %v", node.Name, err)
continue
}
_, err = oc.Registry.GetSubnet(node.Name)
if err == nil {
// subnet already exists, continue
continue
}
err = oc.addNode(node.Name, nodeIP)
if err != nil {
return err
}
}
return nil
}
func (p *pool) RunIfFree(subnet *net.IPNet, cb func() error) error {
p.mu.Lock()
defer p.mu.Unlock()
if _, ok := p.allocated[subnet.String()]; ok {
return nil
}
return cb()
}
func (blf *BlackListFilter) UnBlock(ipnet *net.IPNet) {
blf.lock.Lock()
defer blf.lock.Unlock()
for e := blf.blacklist.Front(); e != nil; e = e.Next() {
if e.Value.(*net.IPNet).String() == ipnet.String() {
blf.blacklist.Remove(e)
return
}
}
}