// initAnycast initialises the anycast configuration. func (e *Engine) initAnycast() { if err := e.ncc.Dial(); err != nil { log.Fatalf("Failed to connect to NCC: %v", err) } defer e.ncc.Close() vips := make([]*seesaw.VIP, 0) if e.config.ClusterVIP.IPv4Addr != nil { for _, ip := range e.config.ServiceAnycastIPv4 { vips = append(vips, seesaw.NewVIP(ip, nil)) } } if e.config.ClusterVIP.IPv6Addr != nil { for _, ip := range e.config.ServiceAnycastIPv6 { vips = append(vips, seesaw.NewVIP(ip, nil)) } } for _, vip := range vips { if err := e.lbInterface.AddVIP(vip); err != nil { log.Fatalf("Failed to add VIP %v: %v", vip, err) } log.Infof("Advertising BGP route for %v", vip) if err := e.ncc.BGPAdvertiseVIP(vip.IP.IP()); err != nil { log.Fatalf("Failed to advertise VIP %v: %v", vip, err) } } }
// down takes down an IP address for a vserver, then takes down all services // for that IP address. func (v *vserver) down(ip seesaw.IP) { ncc := v.engine.ncc if err := ncc.Dial(); err != nil { log.Fatalf("%v: failed to connect to NCC: %v", v, err) } defer ncc.Close() // If this is an anycast VIP, withdraw the BGP route. nip := ip.IP() if seesaw.IsAnycast(nip) { if v.engine.config.AnycastEnabled { log.Infof("%v: withdrawing BGP route for %v", v, ip) if err := ncc.BGPWithdrawVIP(nip); err != nil { log.Fatalf("%v: failed to withdraw VIP %v: %v", v, ip, err) } } vip := seesaw.NewVIP(nip, nil) if err := v.engine.lbInterface.DeleteVIP(vip); err != nil { log.Fatalf("%v: failed to remove VIP %v: %v", v, ip, err) } if err := v.engine.lbInterface.DeleteVserver(v.lbVservers[ip], ip.AF()); err != nil { log.Fatalf("%v: failed to delete Vserver: %v", v, err) } } // TODO(jsing): Should we delay while the BGP routes propagate? delete(v.active, ip) v.updateServices(ip) log.Infof("%v: VIP %v down", v, ip) }
// up brings up all healthy services for an IP address for a vserver, then // brings up the IP address. func (v *vserver) up(ip seesaw.IP) { ncc := v.engine.ncc if err := ncc.Dial(); err != nil { log.Fatalf("%v: failed to connect to NCC: %v", v, err) } defer ncc.Close() v.active[ip] = true v.updateServices(ip) // If this is an anycast VIP, start advertising a BGP route. nip := ip.IP() if seesaw.IsAnycast(nip) { // TODO(jsing): Create an LBVserver that only encapsulates // the necessary state, rather than storing a full vserver // snapshot. lbVserver := v.snapshot() lbVserver.Services = nil lbVserver.Warnings = nil if err := v.engine.lbInterface.AddVserver(lbVserver, ip.AF()); err != nil { log.Fatalf("%v: failed to add Vserver: %v", v, err) } v.lbVservers[ip] = lbVserver vip := seesaw.NewVIP(nip, nil) if err := v.engine.lbInterface.AddVIP(vip); err != nil { log.Fatalf("%v: failed to add VIP %v: %v", v, ip, err) } // TODO(angusc): Filter out anycast VIPs for non-anycast clusters further // upstream. if v.engine.config.AnycastEnabled { log.Infof("%v: advertising BGP route for %v", v, ip) if err := ncc.BGPAdvertiseVIP(nip); err != nil { log.Fatalf("%v: failed to advertise VIP %v: %v", v, ip, err) } } else { log.Warningf("%v: %v is an anycast VIP, but anycast is not enabled", v, ip) } } log.Infof("%v: VIP %v up", v, ip) }
func interfaceTests(ncc client.NCC) { iface := *testIface vip := net.ParseIP(*clusterVIPStr) rtID := uint8(*routingTableID) out, err := exec.Command(ipCmd, "link", "show", "dev", iface).Output() if err != nil { log.Fatalf("Failed to get link state for %v: %v", iface, err) } if !strings.Contains(string(out), " state DOWN ") { log.Fatalf("Link state for %v is not DOWN: %v", iface, string(out)) } out, err = exec.Command(ipCmd, "addr", "show", "dev", iface).Output() if err != nil { log.Fatalf("Failed to get addresses for %v: %v", iface, err) } if !strings.Contains(string(out), fmt.Sprintf(" inet %s/", vip)) { log.Fatalf("VIP address is not configured on interface: %v", string(out)) } out, err = exec.Command(ipCmd, "rule", "show").Output() if err != nil { log.Fatalf("Failed to get routing policy: %v", err) } rpFrom := fmt.Sprintf("from %s lookup %d", vip, rtID) if !strings.Contains(string(out), rpFrom) { log.Fatalf("Routing policy (from) is not correctly configured for %s: %v", vip, string(out)) } rpTo := fmt.Sprintf("from all to %s lookup %d", vip, rtID) if !strings.Contains(string(out), rpTo) { log.Fatalf("Routing policy (to) is not correctly configured for %s: %v", vip, string(out)) } lb := lbInterface(ncc) // Add a unicast VIP. unicastVIP := seesaw.NewVIP(net.ParseIP(*unicastVIPStr), nil) if err := lb.AddVIP(unicastVIP); err != nil { log.Fatalf("Failed to add VIP %v: %v", unicastVIP, err) } out, err = exec.Command(ipCmd, "addr", "show", "dev", iface).Output() if err != nil { log.Fatalf("Failed to get addresses for %v: %v", iface, err) } if !strings.Contains(string(out), fmt.Sprintf(" inet %s/", unicastVIP)) { log.Fatalf("Unicast VIP address is not configured on interface: %v", string(out)) } // Going up... if err := lb.Up(); err != nil { log.Fatalf("Failed to bring LB interface up: %v", err) } out, err = exec.Command(ipCmd, "link", "show", "dev", iface).Output() if err != nil { log.Fatalf("Failed to get link state for %v: %v", iface, err) } if !strings.Contains(string(out), ",UP,") { log.Fatalf("Interface %v is not UP: %v", iface, string(out)) } // Remove unicast VIP. if err := lb.DeleteVIP(unicastVIP); err != nil { log.Fatalf("Failed to remove unicast VIP %v: %v", unicastVIP, err) } out, err = exec.Command(ipCmd, "addr", "show", "dev", iface).Output() if err != nil { log.Fatalf("Failed to get addresses for %v: %v", iface, err) } if strings.Contains(string(out), fmt.Sprintf(" inet %s/", unicastVIP)) { log.Fatalf("Unicast VIP address is still configured on interface: %v", string(out)) } // Add VLAN ipv4Addr, ipv4Net, err := net.ParseCIDR(*vlanNodeIPStr) if err != nil { log.Fatalf("Failed to parse VLAN node IPv4 address: %v", err) } ipv6Addr, ipv6Net, err := net.ParseCIDR(*vlanNodeIPv6Str) if err != nil { log.Fatalf("Failed to parse VLAN node IPv6 address: %v", err) } vlan := &seesaw.VLAN{ ID: uint16(*vlanID), Host: seesaw.Host{ IPv4Addr: ipv4Addr, IPv4Mask: ipv4Net.Mask, IPv6Addr: ipv6Addr, IPv6Mask: ipv6Net.Mask, }, } if err := lb.AddVLAN(vlan); err != nil { log.Fatalf("Failed to create VLAN %d: %v", vlan.ID, err) } vlanIface := fmt.Sprintf("%s.%d", iface, vlan.ID) out, err = exec.Command(ipCmd, "link", "show", "dev", vlanIface).Output() if err != nil { log.Fatalf("Failed to get link state for %v: %v", vlanIface, err) } if !strings.Contains(string(out), ",UP,") { log.Fatalf("Interface %v is not UP: %v", vlanIface, string(out)) } // Add a VIP on the VLAN vlanVIP := seesaw.NewVIP(net.ParseIP(*vlanVIPStr), nil) if err := lb.AddVIP(vlanVIP); err != nil { log.Fatalf("Failed to add VIP %v: %v", vlanVIP, err) } out, err = exec.Command(ipCmd, "addr", "show", "dev", vlanIface).Output() if err != nil { log.Fatalf("Failed to get addresses for %v: %v", vlanIface, err) } if !strings.Contains(string(out), fmt.Sprintf(" inet %s/", vlanVIP)) { log.Fatalf("VLAN VIP address is not configured on interface: %v", string(out)) } // Remove the VIP from the VLAN if err := lb.DeleteVIP(vlanVIP); err != nil { log.Fatalf("Failed to delete VIP %v: %v", vlanVIP, err) } out, err = exec.Command(ipCmd, "addr", "show", "dev", vlanIface).Output() if err != nil { log.Fatalf("Failed to get addresses for %v: %v", vlanIface, err) } if strings.Contains(string(out), fmt.Sprintf("%s", vlanVIP)) { log.Fatalf("VLAN VIP address is still configured on interface: %v", string(out)) } // Remove VLAN if err := lb.DeleteVLAN(vlan); err != nil { log.Fatalf("Failed to delete VLAN %d: %v", vlan.ID, err) } // Going down... if err := lb.Down(); err != nil { log.Fatalf("Failed to bring LB interface down: %v", err) } out, err = exec.Command(ipCmd, "link", "show", "dev", iface).Output() if err != nil { log.Fatalf("Failed to get link state for %v: %v", iface, err) } if !strings.Contains(string(out), " state DOWN ") { log.Fatalf("Link state for %v is not DOWN: %v", iface, string(out)) } }
func addVservers(c *Cluster, p *pb.Cluster) { for _, vs := range p.Vserver { host := vs.GetEntryAddress() v := NewVserver(vs.GetName(), protoToHost(host)) v.Enabled = host.GetStatus() == pb.Host_PRODUCTION || host.GetStatus() == pb.Host_TESTING v.UseFWM = vs.GetUseFwm() v.Warnings = vs.GetWarning() sort.Strings(v.Warnings) for _, ip := range []net.IP{v.Host.IPv4Addr, v.Host.IPv6Addr} { if ip != nil { v.AddVIP(seesaw.NewVIP(ip, c.VIPSubnets)) } } for _, ve := range vs.VserverEntry { var proto seesaw.IPProto switch ve.GetProtocol() { case pb.Protocol_TCP: proto = seesaw.IPProtoTCP case pb.Protocol_UDP: proto = seesaw.IPProtoUDP default: // TODO(angusc): Consider this VServer broken. log.Errorf("%v: Unsupported IP protocol %v", vs.GetName(), ve.GetProtocol()) continue } e := NewVserverEntry(uint16(ve.GetPort()), proto) var scheduler seesaw.LBScheduler switch ve.GetScheduler() { case pb.VserverEntry_RR: scheduler = seesaw.LBSchedulerRR case pb.VserverEntry_WRR: scheduler = seesaw.LBSchedulerWRR case pb.VserverEntry_LC: scheduler = seesaw.LBSchedulerLC case pb.VserverEntry_WLC: scheduler = seesaw.LBSchedulerWLC case pb.VserverEntry_SH: scheduler = seesaw.LBSchedulerSH default: // TODO(angusc): Consider this VServer broken. log.Errorf("%v: Unsupported scheduler %v", vs.GetName(), ve.GetScheduler()) continue } e.Scheduler = scheduler var mode seesaw.LBMode switch ve.GetMode() { case pb.VserverEntry_DSR: mode = seesaw.LBModeDSR case pb.VserverEntry_NAT: mode = seesaw.LBModeNAT default: // TODO(angusc): Consider this VServer broken. log.Errorf("%v: Unsupported mode %v", vs.GetName(), ve.GetMode()) continue } e.Mode = mode e.Persistence = int(ve.GetPersistence()) e.OnePacket = ve.GetOnePacket() e.HighWatermark = ve.GetServerHighWatermark() e.LowWatermark = ve.GetServerLowWatermark() if e.HighWatermark < e.LowWatermark { e.HighWatermark = e.LowWatermark } e.LThreshold = int(ve.GetLthreshold()) e.UThreshold = int(ve.GetUthreshold()) for _, hc := range protosToHealthchecks(ve.Healthcheck, e.Port) { if err := e.AddHealthcheck(hc); err != nil { log.Warning(err) } } if err := v.AddVserverEntry(e); err != nil { log.Warning(err) } } for _, backend := range vs.Backend { status := backend.GetHost().GetStatus() b := &seesaw.Backend{ Host: protoToHost(backend.GetHost()), Weight: backend.GetWeight(), Enabled: status == pb.Host_PRODUCTION || status == pb.Host_TESTING, InService: status != pb.Host_PROPOSED && status != pb.Host_BUILDING, } if err := v.AddBackend(b); err != nil { log.Warning(err) } } for _, hc := range protosToHealthchecks(vs.Healthcheck, 0) { if err := v.AddHealthcheck(hc); err != nil { log.Warning(err) } } if err := c.AddVserver(v); err != nil { log.Warning(err) } } }