func proxy(sm *subnet.SubnetManager, tun *os.File, conn *net.UDPConn, tunMTU uint, port int) {
log.Info("Running slow proxy loop")
rtr := NewRouter(port)
go proxyTunToUdp(rtr, tun, conn, tunMTU)
go proxyUdpToTun(conn, tun, tunMTU)
log.Info("Watching for new subnet leases")
evts := make(chan subnet.EventBatch)
sm.Start(evts)
for evtBatch := range evts {
for _, evt := range evtBatch {
if evt.Type == subnet.SubnetAdded {
log.Info("Subnet added: ", evt.Lease.Network)
var attrs subnet.BaseAttrs
if err := json.Unmarshal([]byte(evt.Lease.Data), &attrs); err != nil {
log.Error("Error decoding subnet lease JSON: ", err)
continue
}
rtr.SetRoute(evt.Lease.Network, attrs.PublicIP)
} else if evt.Type == subnet.SubnetRemoved {
log.Info("Subnet removed: ", evt.Lease.Network)
rtr.DelRoute(evt.Lease.Network)
} else {
log.Error("Internal error: unknown event type: ", int(evt.Type))
}
}
}
}
func Run(sm *subnet.SubnetManager, tepIface *net.Interface, tepIP net.IP, port int, ipMasq bool, ready backend.ReadyFunc) {
sn, err := acquireLease(sm, tepIP)
if err != nil {
log.Error("Failed to acquire lease: ", err)
return
}
tun, tunName, err := ip.OpenTun("rudder%d")
if err != nil {
log.Error("Failed to open TUN device: ", err)
return
}
localAddr := net.UDPAddr{
Port: port,
}
conn, err := net.ListenUDP("udp4", &localAddr)
if err != nil {
log.Error("Failed to start listening on UDP socket: ", err)
return
}
// Interface's subnet is that of the whole overlay network (e.g. /16)
// and not that of the individual host (e.g. /24)
tunNet := ip.IP4Net{
IP: sn.IP,
PrefixLen: sm.GetConfig().Network.PrefixLen,
}
// TUN MTU will be smaller b/c of encap (IP+UDP hdrs)
var mtu int
if tepIface.MTU > 0 {
mtu = tepIface.MTU - encapOverhead
} else {
log.Errorf("Failed to determine MTU for %s interface", tepIP)
return
}
err = configureIface(tunName, tunNet, mtu)
if err != nil {
return
}
if ipMasq {
err = setupIpMasq(tunNet.Network(), tunName)
if err != nil {
return
}
}
// all initialized and ready for business
log.Info("UDP encapsulation initialized")
ready(sn, mtu)
fastProxy(sm, tun, conn, tunNet.IP, uint(mtu), port)
}
func makeSubnetManager() *subnet.SubnetManager {
for {
sm, err := subnet.NewSubnetManager(opts.etcdEndpoint, opts.etcdPrefix)
if err == nil {
return sm
}
log.Error("Failed to create SubnetManager: ", err)
time.Sleep(time.Second)
}
}
func (sm *SubnetManager) AcquireLease(tep ip.IP4, data string) (ip.IP4Net, error) {
for i := 0; i < registerRetries; i++ {
var err error
sm.leases, err = sm.getLeases()
if err != nil {
return ip.IP4Net{}, err
}
// try to reuse a subnet if there's one that match our IP
for _, l := range sm.leases {
var ba BaseAttrs
err = json.Unmarshal([]byte(l.Data), &ba)
if err != nil {
log.Error("Error parsing subnet lease JSON: ", err)
} else {
if tep == ba.PublicIP {
resp, err := sm.registry.updateSubnet(l.Network.StringSep(".", "-"), data, subnetTTL)
if err != nil {
return ip.IP4Net{}, err
}
sm.myLease.Network = l.Network
sm.leaseExp = *resp.Node.Expiration
return l.Network, nil
}
}
}
// no existing match, grab a new one
sn, err := sm.allocateSubnet()
if err != nil {
return ip.IP4Net{}, err
}
resp, err := sm.registry.createSubnet(sn.StringSep(".", "-"), data, subnetTTL)
switch {
case err == nil:
sm.myLease.Network = sn
sm.leaseExp = *resp.Node.Expiration
return sn, nil
// if etcd returned Key Already Exists, try again.
case err.(*etcd.EtcdError).ErrorCode == etcdKeyAlreadyExists:
continue
default:
return ip.IP4Net{}, err
}
}
return ip.IP4Net{}, errors.New("Max retries reached trying to acquire a subnet")
}
func lookupIface() (*net.Interface, net.IP) {
var iface *net.Interface
var tep net.IP
var err error
if len(opts.iface) > 0 {
if tep = net.ParseIP(opts.iface); tep != nil {
iface, err = ip.GetInterfaceByIP(tep)
if err != nil {
log.Errorf("Error looking up interface %s: %s", opts.iface, err)
return nil, nil
}
} else {
iface, err = net.InterfaceByName(opts.iface)
if err != nil {
log.Errorf("Error looking up interface %s: %s", opts.iface, err)
return nil, nil
}
}
} else {
log.Info("Determining IP address of default interface")
for {
if iface, err = ip.GetDefaultGatewayIface(); err == nil {
break
}
log.Error("Failed to get default interface: ", err)
time.Sleep(time.Second)
}
}
if tep == nil {
tep, err = ip.GetIfaceIP4Addr(iface)
if err != nil {
log.Error("Failed to find IPv4 address for interface ", iface.Name)
}
}
return iface, tep
}
func makeSubnetManager() *subnet.SubnetManager {
etcdCli := etcd.NewClient([]string{opts.etcdEndpoint})
for {
sm, err := subnet.NewSubnetManager(etcdCli, opts.etcdPrefix)
if err == nil {
return sm
}
log.Error("Failed to create SubnetManager: ", err)
time.Sleep(time.Second)
}
}
func setupIpMasq(ipn ip.IP4Net, iface string) error {
ipt, err := ip.NewIPTables()
if err != nil {
log.Error("Failed to setup IP Masquerade. iptables was not found")
return err
}
err = ipt.ClearChain("nat", "RUDDER")
if err != nil {
log.Error("Failed to create/clear RUDDER chain in NAT table: ", err)
return err
}
rules := [][]string{
// This rule makes sure we don't NAT traffic within overlay network (e.g. coming out of docker0)
[]string{"RUDDER", "-d", ipn.String(), "-j", "ACCEPT"},
// This rule makes sure we don't NAT multicast traffic within overlay network
[]string{"RUDDER", "-d", "224.0.0.0/4", "-j", "ACCEPT"},
// This rule will NAT everything originating from our overlay network and
[]string{"RUDDER", "!", "-o", iface, "-j", "MASQUERADE"},
// This rule will take everything coming from overlay and sent it to RUDDER chain
[]string{"POSTROUTING", "-s", ipn.String(), "-j", "RUDDER"},
}
for _, args := range rules {
log.Info("Adding iptables rule: ", strings.Join(args, " "))
err = ipt.AppendUnique("nat", args...)
if err != nil {
log.Error("Failed to insert IP masquerade rule: ", err)
return err
}
}
return nil
}
func (sm *SubnetManager) leaseRenewer() {
dur := sm.leaseExp.Sub(time.Now()) - renewMargin
for {
select {
case <-time.After(dur):
resp, err := sm.registry.updateSubnet(sm.myLease.Network.StringSep(".", "-"), sm.myLease.Data, subnetTTL)
if err != nil {
log.Error("Error renewing lease (trying again in 1 min): ", err)
dur = time.Minute
continue
}
sm.leaseExp = *resp.Node.Expiration
log.Info("Lease renewed, new expiration: ", sm.leaseExp)
dur = sm.leaseExp.Sub(time.Now()) - renewMargin
case <-sm.stop:
return
}
}
}
func acquireLease(sm *subnet.SubnetManager, pubIP net.IP) (ip.IP4Net, error) {
attrs := subnet.BaseAttrs{
PublicIP: ip.FromIP(pubIP),
}
data, err := json.Marshal(&attrs)
if err != nil {
return ip.IP4Net{}, err
}
var sn ip.IP4Net
for {
sn, err = sm.AcquireLease(attrs.PublicIP, string(data))
if err == nil {
log.Info("Subnet lease acquired: ", sn)
break
}
log.Error("Failed to acquire subnet: ", err)
time.Sleep(time.Second)
}
return sn, nil
}
func configureIface(ifname string, ipn ip.IP4Net, mtu int) error {
iface, err := net.InterfaceByName(ifname)
if err != nil {
log.Error("Failed to lookup interface ", ifname)
return err
}
n := ipn.ToIPNet()
err = netlink.NetworkLinkAddIp(iface, n.IP, n)
if err != nil {
log.Errorf("Failed to add IP address %s to %s: %s", n.IP, ifname, err)
return err
}
err = netlink.NetworkSetMTU(iface, mtu)
if err != nil {
log.Errorf("Failed to set MTU for %s: %v", ifname, err)
return err
}
err = netlink.NetworkLinkUp(iface)
if err != nil {
log.Errorf("Failed to set interface %s to UP state: %s", ifname, err)
return err
}
// explicitly add a route since there might be a route for a subnet already
// installed by Docker and then it won't get auto added
err = netlink.AddRoute(ipn.Network().String(), "", "", ifname)
if err != nil && err != syscall.EEXIST {
log.Errorf("Failed to add route (%s -> %s): %v", ipn.Network().String(), ifname, err)
return err
}
return nil
}
func fastProxy(sm *subnet.SubnetManager, tun *os.File, conn *net.UDPConn, tunIP ip.IP4, tunMTU uint, port int) {
log.Info("Running fast proxy loop")
c, err := conn.File()
if err != nil {
log.Error("Converting UDPConn to File failed: ", err)
return
}
defer c.Close()
ctl, peerCtl, err := newCtlSockets()
if err != nil {
log.Error("Failed to create control socket: ", err)
return
}
defer ctl.Close()
defer peerCtl.Close()
go runCProxy(tun, c, peerCtl, tunIP, tunMTU)
log.Info("Watching for new subnet leases")
evts := make(chan subnet.EventBatch)
sm.Start(evts)
for evtBatch := range evts {
for _, evt := range evtBatch {
if evt.Type == subnet.SubnetAdded {
log.Info("Subnet added: ", evt.Lease.Network)
var attrs subnet.BaseAttrs
if err := json.Unmarshal([]byte(evt.Lease.Data), &attrs); err != nil {
log.Error("Error decoding subnet lease JSON: ", err)
continue
}
cmd := C.command{
cmd: C.CMD_SET_ROUTE,
dest_net: C.in_addr_t(evt.Lease.Network.IP.NetworkOrder()),
dest_net_len: C.int(evt.Lease.Network.PrefixLen),
next_hop_ip: C.in_addr_t(attrs.PublicIP.NetworkOrder()),
next_hop_port: C.short(port),
}
writeCommand(ctl, &cmd)
} else if evt.Type == subnet.SubnetRemoved {
log.Info("Subnet removed: ", evt.Lease.Network)
cmd := C.command{
cmd: C.CMD_DEL_ROUTE,
dest_net: C.in_addr_t(evt.Lease.Network.IP.NetworkOrder()),
dest_net_len: C.int(evt.Lease.Network.PrefixLen),
}
writeCommand(ctl, &cmd)
} else {
log.Error("Internal error: unknown event type: ", int(evt.Type))
}
}
}
}