// initAnycast initialises the anycast configuration.
func (e *Engine) initAnycast() {
if err := e.ncc.Dial(); err != nil {
log.Fatalf("Failed to connect to NCC: %v", err)
}
defer e.ncc.Close()
vips := make([]*seesaw.VIP, 0)
if e.config.ClusterVIP.IPv4Addr != nil {
for _, ip := range e.config.ServiceAnycastIPv4 {
vips = append(vips, seesaw.NewVIP(ip, nil))
}
}
if e.config.ClusterVIP.IPv6Addr != nil {
for _, ip := range e.config.ServiceAnycastIPv6 {
vips = append(vips, seesaw.NewVIP(ip, nil))
}
}
for _, vip := range vips {
if err := e.lbInterface.AddVIP(vip); err != nil {
log.Fatalf("Failed to add VIP %v: %v", vip, err)
}
log.Infof("Advertising BGP route for %v", vip)
if err := e.ncc.BGPAdvertiseVIP(vip.IP.IP()); err != nil {
log.Fatalf("Failed to advertise VIP %v: %v", vip, err)
}
}
}
// down takes down an IP address for a vserver, then takes down all services
// for that IP address.
func (v *vserver) down(ip seesaw.IP) {
ncc := v.engine.ncc
if err := ncc.Dial(); err != nil {
log.Fatalf("%v: failed to connect to NCC: %v", v, err)
}
defer ncc.Close()
// If this is an anycast VIP, withdraw the BGP route.
nip := ip.IP()
if seesaw.IsAnycast(nip) {
if v.engine.config.AnycastEnabled {
log.Infof("%v: withdrawing BGP route for %v", v, ip)
if err := ncc.BGPWithdrawVIP(nip); err != nil {
log.Fatalf("%v: failed to withdraw VIP %v: %v", v, ip, err)
}
}
vip := seesaw.NewVIP(nip, nil)
if err := v.engine.lbInterface.DeleteVIP(vip); err != nil {
log.Fatalf("%v: failed to remove VIP %v: %v", v, ip, err)
}
if err := v.engine.lbInterface.DeleteVserver(v.lbVservers[ip], ip.AF()); err != nil {
log.Fatalf("%v: failed to delete Vserver: %v", v, err)
}
}
// TODO(jsing): Should we delay while the BGP routes propagate?
delete(v.active, ip)
v.updateServices(ip)
log.Infof("%v: VIP %v down", v, ip)
}
// up brings up all healthy services for an IP address for a vserver, then
// brings up the IP address.
func (v *vserver) up(ip seesaw.IP) {
ncc := v.engine.ncc
if err := ncc.Dial(); err != nil {
log.Fatalf("%v: failed to connect to NCC: %v", v, err)
}
defer ncc.Close()
v.active[ip] = true
v.updateServices(ip)
// If this is an anycast VIP, start advertising a BGP route.
nip := ip.IP()
if seesaw.IsAnycast(nip) {
// TODO(jsing): Create an LBVserver that only encapsulates
// the necessary state, rather than storing a full vserver
// snapshot.
lbVserver := v.snapshot()
lbVserver.Services = nil
lbVserver.Warnings = nil
if err := v.engine.lbInterface.AddVserver(lbVserver, ip.AF()); err != nil {
log.Fatalf("%v: failed to add Vserver: %v", v, err)
}
v.lbVservers[ip] = lbVserver
vip := seesaw.NewVIP(nip, nil)
if err := v.engine.lbInterface.AddVIP(vip); err != nil {
log.Fatalf("%v: failed to add VIP %v: %v", v, ip, err)
}
// TODO(angusc): Filter out anycast VIPs for non-anycast clusters further
// upstream.
if v.engine.config.AnycastEnabled {
log.Infof("%v: advertising BGP route for %v", v, ip)
if err := ncc.BGPAdvertiseVIP(nip); err != nil {
log.Fatalf("%v: failed to advertise VIP %v: %v", v, ip, err)
}
} else {
log.Warningf("%v: %v is an anycast VIP, but anycast is not enabled", v, ip)
}
}
log.Infof("%v: VIP %v up", v, ip)
}
func interfaceTests(ncc client.NCC) {
iface := *testIface
vip := net.ParseIP(*clusterVIPStr)
rtID := uint8(*routingTableID)
out, err := exec.Command(ipCmd, "link", "show", "dev", iface).Output()
if err != nil {
log.Fatalf("Failed to get link state for %v: %v", iface, err)
}
if !strings.Contains(string(out), " state DOWN ") {
log.Fatalf("Link state for %v is not DOWN: %v", iface, string(out))
}
out, err = exec.Command(ipCmd, "addr", "show", "dev", iface).Output()
if err != nil {
log.Fatalf("Failed to get addresses for %v: %v", iface, err)
}
if !strings.Contains(string(out), fmt.Sprintf(" inet %s/", vip)) {
log.Fatalf("VIP address is not configured on interface: %v", string(out))
}
out, err = exec.Command(ipCmd, "rule", "show").Output()
if err != nil {
log.Fatalf("Failed to get routing policy: %v", err)
}
rpFrom := fmt.Sprintf("from %s lookup %d", vip, rtID)
if !strings.Contains(string(out), rpFrom) {
log.Fatalf("Routing policy (from) is not correctly configured for %s: %v", vip, string(out))
}
rpTo := fmt.Sprintf("from all to %s lookup %d", vip, rtID)
if !strings.Contains(string(out), rpTo) {
log.Fatalf("Routing policy (to) is not correctly configured for %s: %v", vip, string(out))
}
lb := lbInterface(ncc)
// Add a unicast VIP.
unicastVIP := seesaw.NewVIP(net.ParseIP(*unicastVIPStr), nil)
if err := lb.AddVIP(unicastVIP); err != nil {
log.Fatalf("Failed to add VIP %v: %v", unicastVIP, err)
}
out, err = exec.Command(ipCmd, "addr", "show", "dev", iface).Output()
if err != nil {
log.Fatalf("Failed to get addresses for %v: %v", iface, err)
}
if !strings.Contains(string(out), fmt.Sprintf(" inet %s/", unicastVIP)) {
log.Fatalf("Unicast VIP address is not configured on interface: %v", string(out))
}
// Going up...
if err := lb.Up(); err != nil {
log.Fatalf("Failed to bring LB interface up: %v", err)
}
out, err = exec.Command(ipCmd, "link", "show", "dev", iface).Output()
if err != nil {
log.Fatalf("Failed to get link state for %v: %v", iface, err)
}
if !strings.Contains(string(out), ",UP,") {
log.Fatalf("Interface %v is not UP: %v", iface, string(out))
}
// Remove unicast VIP.
if err := lb.DeleteVIP(unicastVIP); err != nil {
log.Fatalf("Failed to remove unicast VIP %v: %v", unicastVIP, err)
}
out, err = exec.Command(ipCmd, "addr", "show", "dev", iface).Output()
if err != nil {
log.Fatalf("Failed to get addresses for %v: %v", iface, err)
}
if strings.Contains(string(out), fmt.Sprintf(" inet %s/", unicastVIP)) {
log.Fatalf("Unicast VIP address is still configured on interface: %v", string(out))
}
// Add VLAN
ipv4Addr, ipv4Net, err := net.ParseCIDR(*vlanNodeIPStr)
if err != nil {
log.Fatalf("Failed to parse VLAN node IPv4 address: %v", err)
}
ipv6Addr, ipv6Net, err := net.ParseCIDR(*vlanNodeIPv6Str)
if err != nil {
log.Fatalf("Failed to parse VLAN node IPv6 address: %v", err)
}
vlan := &seesaw.VLAN{
ID: uint16(*vlanID),
Host: seesaw.Host{
IPv4Addr: ipv4Addr,
IPv4Mask: ipv4Net.Mask,
IPv6Addr: ipv6Addr,
IPv6Mask: ipv6Net.Mask,
},
}
if err := lb.AddVLAN(vlan); err != nil {
log.Fatalf("Failed to create VLAN %d: %v", vlan.ID, err)
}
vlanIface := fmt.Sprintf("%s.%d", iface, vlan.ID)
out, err = exec.Command(ipCmd, "link", "show", "dev", vlanIface).Output()
if err != nil {
log.Fatalf("Failed to get link state for %v: %v", vlanIface, err)
}
if !strings.Contains(string(out), ",UP,") {
log.Fatalf("Interface %v is not UP: %v", vlanIface, string(out))
}
// Add a VIP on the VLAN
vlanVIP := seesaw.NewVIP(net.ParseIP(*vlanVIPStr), nil)
if err := lb.AddVIP(vlanVIP); err != nil {
log.Fatalf("Failed to add VIP %v: %v", vlanVIP, err)
}
out, err = exec.Command(ipCmd, "addr", "show", "dev", vlanIface).Output()
if err != nil {
log.Fatalf("Failed to get addresses for %v: %v", vlanIface, err)
}
if !strings.Contains(string(out), fmt.Sprintf(" inet %s/", vlanVIP)) {
log.Fatalf("VLAN VIP address is not configured on interface: %v", string(out))
}
// Remove the VIP from the VLAN
if err := lb.DeleteVIP(vlanVIP); err != nil {
log.Fatalf("Failed to delete VIP %v: %v", vlanVIP, err)
}
out, err = exec.Command(ipCmd, "addr", "show", "dev", vlanIface).Output()
if err != nil {
log.Fatalf("Failed to get addresses for %v: %v", vlanIface, err)
}
if strings.Contains(string(out), fmt.Sprintf("%s", vlanVIP)) {
log.Fatalf("VLAN VIP address is still configured on interface: %v", string(out))
}
// Remove VLAN
if err := lb.DeleteVLAN(vlan); err != nil {
log.Fatalf("Failed to delete VLAN %d: %v", vlan.ID, err)
}
// Going down...
if err := lb.Down(); err != nil {
log.Fatalf("Failed to bring LB interface down: %v", err)
}
out, err = exec.Command(ipCmd, "link", "show", "dev", iface).Output()
if err != nil {
log.Fatalf("Failed to get link state for %v: %v", iface, err)
}
if !strings.Contains(string(out), " state DOWN ") {
log.Fatalf("Link state for %v is not DOWN: %v", iface, string(out))
}
}
func addVservers(c *Cluster, p *pb.Cluster) {
for _, vs := range p.Vserver {
host := vs.GetEntryAddress()
v := NewVserver(vs.GetName(), protoToHost(host))
v.Enabled = host.GetStatus() == pb.Host_PRODUCTION || host.GetStatus() == pb.Host_TESTING
v.UseFWM = vs.GetUseFwm()
v.Warnings = vs.GetWarning()
sort.Strings(v.Warnings)
for _, ip := range []net.IP{v.Host.IPv4Addr, v.Host.IPv6Addr} {
if ip != nil {
v.AddVIP(seesaw.NewVIP(ip, c.VIPSubnets))
}
}
for _, ve := range vs.VserverEntry {
var proto seesaw.IPProto
switch ve.GetProtocol() {
case pb.Protocol_TCP:
proto = seesaw.IPProtoTCP
case pb.Protocol_UDP:
proto = seesaw.IPProtoUDP
default:
// TODO(angusc): Consider this VServer broken.
log.Errorf("%v: Unsupported IP protocol %v", vs.GetName(), ve.GetProtocol())
continue
}
e := NewVserverEntry(uint16(ve.GetPort()), proto)
var scheduler seesaw.LBScheduler
switch ve.GetScheduler() {
case pb.VserverEntry_RR:
scheduler = seesaw.LBSchedulerRR
case pb.VserverEntry_WRR:
scheduler = seesaw.LBSchedulerWRR
case pb.VserverEntry_LC:
scheduler = seesaw.LBSchedulerLC
case pb.VserverEntry_WLC:
scheduler = seesaw.LBSchedulerWLC
case pb.VserverEntry_SH:
scheduler = seesaw.LBSchedulerSH
default:
// TODO(angusc): Consider this VServer broken.
log.Errorf("%v: Unsupported scheduler %v", vs.GetName(), ve.GetScheduler())
continue
}
e.Scheduler = scheduler
var mode seesaw.LBMode
switch ve.GetMode() {
case pb.VserverEntry_DSR:
mode = seesaw.LBModeDSR
case pb.VserverEntry_NAT:
mode = seesaw.LBModeNAT
default:
// TODO(angusc): Consider this VServer broken.
log.Errorf("%v: Unsupported mode %v", vs.GetName(), ve.GetMode())
continue
}
e.Mode = mode
e.Persistence = int(ve.GetPersistence())
e.OnePacket = ve.GetOnePacket()
e.HighWatermark = ve.GetServerHighWatermark()
e.LowWatermark = ve.GetServerLowWatermark()
if e.HighWatermark < e.LowWatermark {
e.HighWatermark = e.LowWatermark
}
e.LThreshold = int(ve.GetLthreshold())
e.UThreshold = int(ve.GetUthreshold())
for _, hc := range protosToHealthchecks(ve.Healthcheck, e.Port) {
if err := e.AddHealthcheck(hc); err != nil {
log.Warning(err)
}
}
if err := v.AddVserverEntry(e); err != nil {
log.Warning(err)
}
}
for _, backend := range vs.Backend {
status := backend.GetHost().GetStatus()
b := &seesaw.Backend{
Host: protoToHost(backend.GetHost()),
Weight: backend.GetWeight(),
Enabled: status == pb.Host_PRODUCTION || status == pb.Host_TESTING,
InService: status != pb.Host_PROPOSED && status != pb.Host_BUILDING,
}
if err := v.AddBackend(b); err != nil {
log.Warning(err)
}
}
for _, hc := range protosToHealthchecks(vs.Healthcheck, 0) {
if err := v.AddHealthcheck(hc); err != nil {
log.Warning(err)
}
}
if err := c.AddVserver(v); err != nil {
log.Warning(err)
}
}
}