// updateState updates the state of an IP for a vserver based on the state of
// that IP's services.
func (v *vserver) updateState(ip seesaw.IP) {
// A vserver anycast IP is healthy if *all* services for that IP are healthy.
// A vserver unicast IP is healthy if *any* services for that IP are healthy.
var healthy bool
for _, s := range v.services {
if !s.ip.Equal(ip) {
continue
}
healthy = s.healthy
if !healthy && seesaw.IsAnycast(ip.IP()) {
break
}
if healthy && !seesaw.IsAnycast(ip.IP()) {
break
}
}
if v.active[ip] == healthy {
v.updateServices(ip)
return
}
switch {
case !healthy && v.active[ip]:
v.down(ip)
case healthy && !v.active[ip]:
v.up(ip)
}
}
// down takes down an IP address for a vserver, then takes down all services
// for that IP address.
func (v *vserver) down(ip seesaw.IP) {
ncc := v.engine.ncc
if err := ncc.Dial(); err != nil {
log.Fatalf("%v: failed to connect to NCC: %v", v, err)
}
defer ncc.Close()
// If this is an anycast VIP, withdraw the BGP route.
nip := ip.IP()
if seesaw.IsAnycast(nip) {
if v.engine.config.AnycastEnabled {
log.Infof("%v: withdrawing BGP route for %v", v, ip)
if err := ncc.BGPWithdrawVIP(nip); err != nil {
log.Fatalf("%v: failed to withdraw VIP %v: %v", v, ip, err)
}
}
vip := seesaw.NewVIP(nip, nil)
if err := v.engine.lbInterface.DeleteVIP(vip); err != nil {
log.Fatalf("%v: failed to remove VIP %v: %v", v, ip, err)
}
if err := v.engine.lbInterface.DeleteVserver(v.lbVservers[ip], ip.AF()); err != nil {
log.Fatalf("%v: failed to delete Vserver: %v", v, err)
}
}
// TODO(jsing): Should we delay while the BGP routes propagate?
delete(v.active, ip)
v.updateServices(ip)
log.Infof("%v: VIP %v down", v, ip)
}
// up brings up all healthy services for an IP address for a vserver, then
// brings up the IP address.
func (v *vserver) up(ip seesaw.IP) {
ncc := v.engine.ncc
if err := ncc.Dial(); err != nil {
log.Fatalf("%v: failed to connect to NCC: %v", v, err)
}
defer ncc.Close()
v.active[ip] = true
v.updateServices(ip)
// If this is an anycast VIP, start advertising a BGP route.
nip := ip.IP()
if seesaw.IsAnycast(nip) {
// TODO(jsing): Create an LBVserver that only encapsulates
// the necessary state, rather than storing a full vserver
// snapshot.
lbVserver := v.snapshot()
lbVserver.Services = nil
lbVserver.Warnings = nil
if err := v.engine.lbInterface.AddVserver(lbVserver, ip.AF()); err != nil {
log.Fatalf("%v: failed to add Vserver: %v", v, err)
}
v.lbVservers[ip] = lbVserver
vip := seesaw.NewVIP(nip, nil)
if err := v.engine.lbInterface.AddVIP(vip); err != nil {
log.Fatalf("%v: failed to add VIP %v: %v", v, ip, err)
}
// TODO(angusc): Filter out anycast VIPs for non-anycast clusters further
// upstream.
if v.engine.config.AnycastEnabled {
log.Infof("%v: advertising BGP route for %v", v, ip)
if err := ncc.BGPAdvertiseVIP(nip); err != nil {
log.Fatalf("%v: failed to advertise VIP %v: %v", v, ip, err)
}
} else {
log.Warningf("%v: %v is an anycast VIP, but anycast is not enabled", v, ip)
}
}
log.Infof("%v: VIP %v up", v, ip)
}
func main() {
flag.Parse()
cfg, err := conf.ReadConfigFile(*configFile)
if err != nil {
log.Exitf("Failed to read configuration file: %v", err)
}
clusterName := cfgOpt(cfg, "cluster", "name")
if clusterName == "" {
log.Exit("Unable to get cluster name")
}
anycastEnabled := config.DefaultEngineConfig().AnycastEnabled
if opt := cfgOpt(cfg, "cluster", "anycast_enabled"); opt != "" {
if anycastEnabled, err = cfg.GetBool("cluster", "anycast_enabled"); err != nil {
log.Exitf("Unable to parse cluster anycast_enabled: %v", err)
}
}
clusterVIPv4, err := cfgIP(cfg, "cluster", "vip_ipv4")
if err != nil {
log.Exitf("Unable to get cluster vip_ipv4: %v", err)
}
clusterVIPv6, err := cfgIP(cfg, "cluster", "vip_ipv6")
if err != nil {
log.Exitf("Unable to get cluster vip_ipv6: %v", err)
}
nodeIPv4, err := cfgIP(cfg, "cluster", "node_ipv4")
if err != nil {
log.Exitf("Unable to get cluster node_ipv4: %v", err)
}
nodeIPv6, err := cfgIP(cfg, "cluster", "node_ipv6")
if err != nil {
log.Exitf("Unable to get cluster node_ipv6: %v", err)
}
peerIPv4, err := cfgIP(cfg, "cluster", "peer_ipv4")
if err != nil {
log.Exitf("Unable to get cluster peer_ipv4: %v", err)
}
peerIPv6, err := cfgIP(cfg, "cluster", "peer_ipv6")
if err != nil {
log.Exitf("Unable to get cluster peer_ipv6: %v", err)
}
// The default VRID may be overridden via the config file.
vrid := config.DefaultEngineConfig().VRID
if cfg.HasOption("cluster", "vrid") {
id, err := cfg.GetInt("cluster", "vrid")
if err != nil {
log.Exitf("Unable to get VRID: %v", err)
}
if id < 1 || id > 255 {
log.Exitf("Invalid VRID %d - must be between 1 and 255 inclusive", id)
}
vrid = uint8(id)
}
// Optional primary, secondary and tertiary configuration servers.
configServers := make([]string, 0)
for _, level := range []string{"primary", "secondary", "tertiary"} {
if server := cfgOpt(cfg, "config_server", level); server != "" {
configServers = append(configServers, server)
}
}
if len(configServers) == 0 {
configServers = config.DefaultEngineConfig().ConfigServers
}
nodeInterface := config.DefaultEngineConfig().NodeInterface
if opt := cfgOpt(cfg, "interface", "node"); opt != "" {
nodeInterface = opt
}
lbInterface := config.DefaultEngineConfig().LBInterface
if opt := cfgOpt(cfg, "interface", "lb"); opt != "" {
lbInterface = opt
}
// Additional anycast addresses.
serviceAnycastIPv4 := config.DefaultEngineConfig().ServiceAnycastIPv4
serviceAnycastIPv6 := config.DefaultEngineConfig().ServiceAnycastIPv6
if cfg.HasSection("extra_service_anycast") {
opts, err := cfg.GetOptions("extra_service_anycast")
if err != nil {
log.Exitf("Unable to get extra_serivce_anycast options: %v", err)
}
for _, opt := range opts {
ip, err := cfgIP(cfg, "extra_service_anycast", opt)
if err != nil {
log.Exitf("Unable to get extra_service_anycast option %q: %v", opt, err)
}
if !seesaw.IsAnycast(ip) {
log.Exitf("%q is not an anycast address", ip)
}
if ip.To4() != nil {
serviceAnycastIPv4 = append(serviceAnycastIPv4, ip)
} else {
serviceAnycastIPv6 = append(serviceAnycastIPv6, ip)
}
}
}
// Override some of the defaults.
engineCfg := config.DefaultEngineConfig()
engineCfg.AnycastEnabled = anycastEnabled
engineCfg.ConfigFile = *configFile
engineCfg.ConfigServers = configServers
engineCfg.ClusterFile = *clusterFile
engineCfg.ClusterName = clusterName
engineCfg.ClusterVIP.IPv4Addr = clusterVIPv4
engineCfg.ClusterVIP.IPv6Addr = clusterVIPv6
engineCfg.LBInterface = lbInterface
engineCfg.NCCSocket = *nccSocket
engineCfg.Node.IPv4Addr = nodeIPv4
engineCfg.Node.IPv6Addr = nodeIPv6
engineCfg.NodeInterface = nodeInterface
engineCfg.Peer.IPv4Addr = peerIPv4
engineCfg.Peer.IPv6Addr = peerIPv6
engineCfg.ServiceAnycastIPv4 = serviceAnycastIPv4
engineCfg.ServiceAnycastIPv6 = serviceAnycastIPv6
engineCfg.SocketPath = *socketPath
engineCfg.VRID = vrid
// Gentlemen, start your engines...
engine := engine.NewEngine(&engineCfg)
server.ShutdownHandler(engine)
server.ServerRunDirectory("engine", 0, 0)
// TODO(jsing): Drop privileges before starting engine.
engine.Run()
}
func TestReIPVserver(t *testing.T) {
e := newTestEngine()
vserver := newTestVserver(e)
lbIF := e.lbInterface.(*dummyLBInterface)
clusterName := "au-syd"
serviceName := "dns.resolver@au-syd"
tests := []struct {
desc string
file string
health healthcheck.State
wantIPs []string
}{
{desc: "initial", file: "re-ip/config_1.pb", wantIPs: []string{"192.168.36.1"}},
{desc: "re-ip unicast", file: "re-ip/config_2.pb", wantIPs: []string{"192.168.36.5"}},
{desc: "unicast unhealthy", health: healthcheck.StateUnhealthy, wantIPs: []string{"192.168.36.5"}},
{desc: "unicast disabled", file: "re-ip/config_3.pb", wantIPs: nil},
{desc: "re-ip anycast (unhealthy)", file: "re-ip/config_4.pb", wantIPs: nil},
{desc: "anycast health", health: healthcheck.StateHealthy, wantIPs: []string{"192.168.255.1"}},
{desc: "anycast disabled", file: "re-ip/config_5.pb", wantIPs: nil},
{desc: "back to unicast", file: "re-ip/config_1.pb", wantIPs: []string{"192.168.36.1"}},
}
for _, tc := range tests {
log.Infof("Applying config: %s", tc.desc)
if tc.file != "" {
if err := applyConfig(vserver, tc.file, clusterName, serviceName); err != nil {
t.Fatalf("Failed to apply configuration: %v", err)
}
}
if tc.health != healthcheck.StateUnknown {
for k := range vserver.checks {
n := &checkNotification{
key: k,
description: fmt.Sprintf("forced health=%v for step %q", tc.health, tc.desc),
status: healthcheck.Status{State: tc.health},
}
vserver.handleCheckNotification(n)
}
}
wantVIPs := make(map[string]bool)
for _, s := range tc.wantIPs {
wantVIPs[s] = true
}
if seesaw.IsAnycast(vserver.config.IPv4Addr) {
// vserver.vips only tracks unicast VIPs today, so we can only check that
// there are none.
if len(vserver.vips) > 0 {
t.Errorf("vserver(%q).vips = %v, wanted no unicast vips", tc.desc, vserver.vips)
}
} else {
gotVIPs := make(map[string]bool)
for v := range vserver.vips {
gotVIPs[v.IP.String()] = true
}
if diff := pretty.Compare(wantVIPs, gotVIPs); diff != "" {
t.Errorf("vserver(%q).vips unexpected (-want +got):\n%s", tc.desc, diff)
}
}
gotLBVS := make(map[string]bool)
for v := range vserver.lbVservers {
gotLBVS[v.String()] = true
}
if diff := pretty.Compare(wantVIPs, gotLBVS); diff != "" {
t.Errorf("vserver(%q).lbVservers (-want +got):\n%s", tc.desc, diff)
}
gotIFIPs := make(map[string]bool)
for v := range lbIF.vips {
gotIFIPs[v.IP.String()] = true
}
if diff := pretty.Compare(wantVIPs, gotIFIPs); diff != "" {
t.Errorf("After %s, unexpected IPs on the LB interface (-want +got):\n%s", tc.desc, diff)
}
}
}
