// checkStaticIP reserves a static IP allocated to the Forwarding Rule.
func (l *L7) checkStaticIP() (err error) {
if l.fw == nil || l.fw.IPAddress == "" {
return fmt.Errorf("Will not create static IP without a forwarding rule.")
}
// Don't manage staticIPs if the user has specified an IP.
if address, manageStaticIP := l.getEffectiveIP(); !manageStaticIP {
glog.V(3).Infof("Not managing user specified static IP %v", address)
return nil
}
staticIPName := l.namer.Truncate(fmt.Sprintf("%v-%v", forwardingRulePrefix, l.Name))
ip, _ := l.cloud.GetGlobalStaticIP(staticIPName)
if ip == nil {
glog.Infof("Creating static ip %v", staticIPName)
ip, err = l.cloud.ReserveGlobalStaticIP(staticIPName, l.fw.IPAddress)
if err != nil {
if utils.IsHTTPErrorCode(err, http.StatusConflict) ||
utils.IsHTTPErrorCode(err, http.StatusBadRequest) {
glog.V(3).Infof("IP %v(%v) is already reserved, assuming it is OK to use.",
l.fw.IPAddress, staticIPName)
return nil
}
return err
}
}
l.ip = ip
return nil
}
// Cleanup deletes resources specific to this l7 in the right order.
// forwarding rule -> target proxy -> url map
// This leaves backends and health checks, which are shared across loadbalancers.
func (l *L7) Cleanup() error {
if l.fw != nil {
glog.Infof("Deleting global forwarding rule %v", l.fw.Name)
if err := l.cloud.DeleteGlobalForwardingRule(l.fw.Name); err != nil {
if !utils.IsHTTPErrorCode(err, http.StatusNotFound) {
return err
}
}
l.fw = nil
}
if l.tp != nil {
glog.Infof("Deleting target proxy %v", l.tp.Name)
if err := l.cloud.DeleteTargetHttpProxy(l.tp.Name); err != nil {
if !utils.IsHTTPErrorCode(err, http.StatusNotFound) {
return err
}
}
l.tp = nil
}
if l.um != nil {
glog.Infof("Deleting url map %v", l.um.Name)
if err := l.cloud.DeleteUrlMap(l.um.Name); err != nil {
if !utils.IsHTTPErrorCode(err, http.StatusNotFound) {
return err
}
}
l.um = nil
}
return nil
}
func (l *L7) checkForwardingRule(name, proxyLink, ip, portRange string) (fw *compute.ForwardingRule, err error) {
fw, _ = l.cloud.GetGlobalForwardingRule(name)
if fw != nil && (ip != "" && fw.IPAddress != ip || fw.PortRange != portRange) {
glog.Warningf("Recreating forwarding rule %v(%v), so it has %v(%v)",
fw.IPAddress, fw.PortRange, ip, portRange)
if err := l.cloud.DeleteGlobalForwardingRule(name); err != nil {
if !utils.IsHTTPErrorCode(err, http.StatusNotFound) {
return nil, err
}
}
fw = nil
}
if fw == nil {
parts := strings.Split(proxyLink, "/")
glog.Infof("Creating forwarding rule for proxy %v and ip %v:%v", parts[len(parts)-1:], ip, portRange)
fw, err = l.cloud.CreateGlobalForwardingRule(proxyLink, ip, name, portRange)
if err != nil {
return nil, err
}
}
// TODO: If the port range and protocol don't match, recreate the rule
if utils.CompareLinks(fw.Target, proxyLink) {
glog.V(3).Infof("Forwarding rule %v already exists", fw.Name)
} else {
glog.Infof("Forwarding rule %v has the wrong proxy, setting %v overwriting %v",
fw.Name, fw.Target, proxyLink)
if err := l.cloud.SetProxyForGlobalForwardingRule(fw, proxyLink); err != nil {
return nil, err
}
}
return fw, nil
}
func getGCEClient() *gce.GCECloud {
// Creating the cloud interface involves resolving the metadata server to get
// an oauth token. If this fails, the token provider assumes it's not on GCE.
// No errors are thrown. So we need to keep retrying till it works because
// we know we're on GCE.
for {
cloudInterface, err := cloudprovider.GetCloudProvider("gce", nil)
if err == nil {
cloud := cloudInterface.(*gce.GCECloud)
// If this controller is scheduled on a node without compute/rw
// it won't be allowed to list backends. We can assume that the
// user has no need for Ingress in this case. If they grant
// permissions to the node they will have to restart the controller
// manually to re-create the client.
if _, err = cloud.ListBackendServices(); err == nil || utils.IsHTTPErrorCode(err, http.StatusForbidden) {
return cloud
}
glog.Warningf("Failed to list backend services, retrying: %v", err)
} else {
glog.Warningf("Failed to retrieve cloud interface, retrying: %v", err)
}
time.Sleep(cloudClientRetryInterval)
}
}
// Delete deletes the health check by port.
func (h *HealthChecks) Delete(port int64) error {
name := h.namer.BeName(port)
glog.Infof("Deleting health check %v", name)
if err := h.cloud.DeleteHttpHealthCheck(h.namer.BeName(port)); err != nil {
if !utils.IsHTTPErrorCode(err, http.StatusNotFound) {
return err
}
}
return nil
}
func (l *L7) deleteOldSSLCert() (err error) {
if l.oldSSLCert == nil || l.sslCert == nil || l.oldSSLCert.Name == l.sslCert.Name {
return nil
}
glog.Infof("Cleaning up old SSL Certificate %v, current name %v", l.oldSSLCert.Name, l.sslCert.Name)
if err := l.cloud.DeleteSslCertificate(l.oldSSLCert.Name); err != nil {
if !utils.IsHTTPErrorCode(err, http.StatusNotFound) {
return err
}
}
return nil
}
// Delete deletes the Backend for the given port.
func (b *Backends) Delete(port int64) (err error) {
name := b.namer.BeName(port)
glog.Infof("Deleting backend %v", name)
defer func() {
if utils.IsHTTPErrorCode(err, http.StatusNotFound) {
err = nil
}
if err == nil {
b.snapshotter.Delete(portKey(port))
}
}()
// Try deleting health checks even if a backend is not found.
if err = b.cloud.DeleteBackendService(name); err != nil &&
!utils.IsHTTPErrorCode(err, http.StatusNotFound) {
return err
}
if err = b.healthChecker.Delete(port); err != nil &&
!utils.IsHTTPErrorCode(err, http.StatusNotFound) {
return err
}
return nil
}
// IsHealthy returns an error if the cluster manager is unhealthy.
func (c *ClusterManager) IsHealthy() (err error) {
// TODO: Expand on this, for now we just want to detect when the GCE client
// is broken.
_, err = c.backendPool.List()
// If this container is scheduled on a node without compute/rw it is
// effectively useless, but it is healthy. Reporting it as unhealthy
// will lead to container crashlooping.
if utils.IsHTTPErrorCode(err, http.StatusForbidden) {
glog.Infof("Reporting cluster as healthy, but unable to list backends: %v", err)
return nil
}
return
}
// Sync syncs kubernetes instances with the instances in the instance group.
func (i *Instances) Sync(nodes []string) (err error) {
glog.V(3).Infof("Syncing nodes %v", nodes)
defer func() {
// The node pool is only responsible for syncing nodes to instance
// groups. It never creates/deletes, so if an instance groups is
// not found there's nothing it can do about it anyway. Most cases
// this will happen because the backend pool has deleted the instance
// group, however if it happens because a user deletes the IG by mistake
// we should just wait till the backend pool fixes it.
if utils.IsHTTPErrorCode(err, http.StatusNotFound) {
glog.Infof("Node pool encountered a 404, ignoring: %v", err)
err = nil
}
}()
pool := i.snapshotter.Snapshot()
for name := range pool {
gceNodes := sets.NewString()
gceNodes, err = i.list(name)
if err != nil {
return err
}
kubeNodes := sets.NewString(nodes...)
// A node deleted via kubernetes could still exist as a gce vm. We don't
// want to route requests to it. Similarly, a node added to kubernetes
// needs to get added to the instance group so we do route requests to it.
removeNodes := gceNodes.Difference(kubeNodes).List()
addNodes := kubeNodes.Difference(gceNodes).List()
if len(removeNodes) != 0 {
if err = i.Remove(
name, gceNodes.Difference(kubeNodes).List()); err != nil {
return err
}
}
if len(addNodes) != 0 {
if err = i.Add(
name, kubeNodes.Difference(gceNodes).List()); err != nil {
return err
}
}
}
return nil
}
// Cleanup deletes resources specific to this l7 in the right order.
// forwarding rule -> target proxy -> url map
// This leaves backends and health checks, which are shared across loadbalancers.
func (l *L7) Cleanup() error {
if l.fw != nil {
glog.Infof("Deleting global forwarding rule %v", l.fw.Name)
if err := l.cloud.DeleteGlobalForwardingRule(l.fw.Name); err != nil {
if !utils.IsHTTPErrorCode(err, http.StatusNotFound) {
return err
}
}
l.fw = nil
}
if l.fws != nil {
glog.Infof("Deleting global forwarding rule %v", l.fws.Name)
if err := l.cloud.DeleteGlobalForwardingRule(l.fws.Name); err != nil {
if !utils.IsHTTPErrorCode(err, http.StatusNotFound) {
return err
}
l.fws = nil
}
}
if l.ip != nil {
glog.Infof("Deleting static IP %v(%v)", l.ip.Name, l.ip.Address)
if err := l.cloud.DeleteGlobalStaticIP(l.ip.Name); err != nil {
if !utils.IsHTTPErrorCode(err, http.StatusNotFound) {
return err
}
l.ip = nil
}
}
if l.tps != nil {
glog.Infof("Deleting target https proxy %v", l.tps.Name)
if err := l.cloud.DeleteTargetHttpsProxy(l.tps.Name); err != nil {
if !utils.IsHTTPErrorCode(err, http.StatusNotFound) {
return err
}
}
l.tps = nil
}
if l.sslCert != nil {
glog.Infof("Deleting sslcert %v", l.sslCert.Name)
if err := l.cloud.DeleteSslCertificate(l.sslCert.Name); err != nil {
if !utils.IsHTTPErrorCode(err, http.StatusNotFound) {
return err
}
}
l.sslCert = nil
}
if l.tp != nil {
glog.Infof("Deleting target http proxy %v", l.tp.Name)
if err := l.cloud.DeleteTargetHttpProxy(l.tp.Name); err != nil {
if !utils.IsHTTPErrorCode(err, http.StatusNotFound) {
return err
}
}
l.tp = nil
}
if l.um != nil {
glog.Infof("Deleting url map %v", l.um.Name)
if err := l.cloud.DeleteUrlMap(l.um.Name); err != nil {
if !utils.IsHTTPErrorCode(err, http.StatusNotFound) {
return err
}
}
l.um = nil
}
return nil
}
// sync manages Ingress create/updates/deletes.
func (lbc *LoadBalancerController) sync(key string) (err error) {
if !lbc.hasSynced() {
time.Sleep(storeSyncPollPeriod)
return fmt.Errorf("Waiting for stores to sync")
}
glog.V(3).Infof("Syncing %v", key)
paths, err := lbc.ingLister.List()
if err != nil {
return err
}
nodePorts := lbc.tr.toNodePorts(&paths)
lbNames := lbc.ingLister.Store.ListKeys()
lbs, err := lbc.ListRuntimeInfo()
if err != nil {
return err
}
nodeNames, err := lbc.getReadyNodeNames()
if err != nil {
return err
}
obj, ingExists, err := lbc.ingLister.Store.GetByKey(key)
if err != nil {
return err
}
// This performs a 2 phase checkpoint with the cloud:
// * Phase 1 creates/verifies resources are as expected. At the end of a
// successful checkpoint we know that existing L7s are WAI, and the L7
// for the Ingress associated with "key" is ready for a UrlMap update.
// If this encounters an error, eg for quota reasons, we want to invoke
// Phase 2 right away and retry checkpointing.
// * Phase 2 performs GC by refcounting shared resources. This needs to
// happen periodically whether or not stage 1 fails. At the end of a
// successful GC we know that there are no dangling cloud resources that
// don't have an associated Kubernetes Ingress/Service/Endpoint.
defer func() {
if deferErr := lbc.CloudClusterManager.GC(lbNames, nodePorts); deferErr != nil {
err = fmt.Errorf("Error during sync %v, error during GC %v", err, deferErr)
}
glog.V(3).Infof("Finished syncing %v", key)
}()
// Record any errors during sync and throw a single error at the end. This
// allows us to free up associated cloud resources ASAP.
var syncError error
if err := lbc.CloudClusterManager.Checkpoint(lbs, nodeNames, nodePorts); err != nil {
// TODO: Implement proper backoff for the queue.
eventMsg := "GCE"
if utils.IsHTTPErrorCode(err, http.StatusForbidden) {
eventMsg += " :Quota"
}
if ingExists {
lbc.recorder.Eventf(obj.(*extensions.Ingress), api.EventTypeWarning, eventMsg, err.Error())
} else {
err = fmt.Errorf("%v Error: %v", eventMsg, err)
}
syncError = err
}
if !ingExists {
return syncError
}
// Update the UrlMap of the single loadbalancer that came through the watch.
l7, err := lbc.CloudClusterManager.l7Pool.Get(key)
if err != nil {
return fmt.Errorf("%v, unable to get loadbalancer: %v", syncError, err)
}
ing := *obj.(*extensions.Ingress)
if urlMap, err := lbc.tr.toUrlMap(&ing); err != nil {
syncError = fmt.Errorf("%v, convert to url map error %v", syncError, err)
} else if err := l7.UpdateUrlMap(urlMap); err != nil {
lbc.recorder.Eventf(&ing, api.EventTypeWarning, "UrlMap", err.Error())
syncError = fmt.Errorf("%v, update url map error: %v", syncError, err)
} else if err := lbc.updateIngressStatus(l7, ing); err != nil {
lbc.recorder.Eventf(&ing, api.EventTypeWarning, "Status", err.Error())
syncError = fmt.Errorf("%v, update ingress error: %v", syncError, err)
}
return syncError
}