// checkStaticIP reserves a static IP allocated to the Forwarding Rule. func (l *L7) checkStaticIP() (err error) { if l.fw == nil || l.fw.IPAddress == "" { return fmt.Errorf("Will not create static IP without a forwarding rule.") } // Don't manage staticIPs if the user has specified an IP. if address, manageStaticIP := l.getEffectiveIP(); !manageStaticIP { glog.V(3).Infof("Not managing user specified static IP %v", address) return nil } staticIPName := l.namer.Truncate(fmt.Sprintf("%v-%v", forwardingRulePrefix, l.Name)) ip, _ := l.cloud.GetGlobalStaticIP(staticIPName) if ip == nil { glog.Infof("Creating static ip %v", staticIPName) ip, err = l.cloud.ReserveGlobalStaticIP(staticIPName, l.fw.IPAddress) if err != nil { if utils.IsHTTPErrorCode(err, http.StatusConflict) || utils.IsHTTPErrorCode(err, http.StatusBadRequest) { glog.V(3).Infof("IP %v(%v) is already reserved, assuming it is OK to use.", l.fw.IPAddress, staticIPName) return nil } return err } } l.ip = ip return nil }
// Cleanup deletes resources specific to this l7 in the right order. // forwarding rule -> target proxy -> url map // This leaves backends and health checks, which are shared across loadbalancers. func (l *L7) Cleanup() error { if l.fw != nil { glog.Infof("Deleting global forwarding rule %v", l.fw.Name) if err := l.cloud.DeleteGlobalForwardingRule(l.fw.Name); err != nil { if !utils.IsHTTPErrorCode(err, http.StatusNotFound) { return err } } l.fw = nil } if l.tp != nil { glog.Infof("Deleting target proxy %v", l.tp.Name) if err := l.cloud.DeleteTargetHttpProxy(l.tp.Name); err != nil { if !utils.IsHTTPErrorCode(err, http.StatusNotFound) { return err } } l.tp = nil } if l.um != nil { glog.Infof("Deleting url map %v", l.um.Name) if err := l.cloud.DeleteUrlMap(l.um.Name); err != nil { if !utils.IsHTTPErrorCode(err, http.StatusNotFound) { return err } } l.um = nil } return nil }
func (l *L7) checkForwardingRule(name, proxyLink, ip, portRange string) (fw *compute.ForwardingRule, err error) { fw, _ = l.cloud.GetGlobalForwardingRule(name) if fw != nil && (ip != "" && fw.IPAddress != ip || fw.PortRange != portRange) { glog.Warningf("Recreating forwarding rule %v(%v), so it has %v(%v)", fw.IPAddress, fw.PortRange, ip, portRange) if err := l.cloud.DeleteGlobalForwardingRule(name); err != nil { if !utils.IsHTTPErrorCode(err, http.StatusNotFound) { return nil, err } } fw = nil } if fw == nil { parts := strings.Split(proxyLink, "/") glog.Infof("Creating forwarding rule for proxy %v and ip %v:%v", parts[len(parts)-1:], ip, portRange) fw, err = l.cloud.CreateGlobalForwardingRule(proxyLink, ip, name, portRange) if err != nil { return nil, err } } // TODO: If the port range and protocol don't match, recreate the rule if utils.CompareLinks(fw.Target, proxyLink) { glog.V(3).Infof("Forwarding rule %v already exists", fw.Name) } else { glog.Infof("Forwarding rule %v has the wrong proxy, setting %v overwriting %v", fw.Name, fw.Target, proxyLink) if err := l.cloud.SetProxyForGlobalForwardingRule(fw, proxyLink); err != nil { return nil, err } } return fw, nil }
func getGCEClient() *gce.GCECloud { // Creating the cloud interface involves resolving the metadata server to get // an oauth token. If this fails, the token provider assumes it's not on GCE. // No errors are thrown. So we need to keep retrying till it works because // we know we're on GCE. for { cloudInterface, err := cloudprovider.GetCloudProvider("gce", nil) if err == nil { cloud := cloudInterface.(*gce.GCECloud) // If this controller is scheduled on a node without compute/rw // it won't be allowed to list backends. We can assume that the // user has no need for Ingress in this case. If they grant // permissions to the node they will have to restart the controller // manually to re-create the client. if _, err = cloud.ListBackendServices(); err == nil || utils.IsHTTPErrorCode(err, http.StatusForbidden) { return cloud } glog.Warningf("Failed to list backend services, retrying: %v", err) } else { glog.Warningf("Failed to retrieve cloud interface, retrying: %v", err) } time.Sleep(cloudClientRetryInterval) } }
// Delete deletes the health check by port. func (h *HealthChecks) Delete(port int64) error { name := h.namer.BeName(port) glog.Infof("Deleting health check %v", name) if err := h.cloud.DeleteHttpHealthCheck(h.namer.BeName(port)); err != nil { if !utils.IsHTTPErrorCode(err, http.StatusNotFound) { return err } } return nil }
func (l *L7) deleteOldSSLCert() (err error) { if l.oldSSLCert == nil || l.sslCert == nil || l.oldSSLCert.Name == l.sslCert.Name { return nil } glog.Infof("Cleaning up old SSL Certificate %v, current name %v", l.oldSSLCert.Name, l.sslCert.Name) if err := l.cloud.DeleteSslCertificate(l.oldSSLCert.Name); err != nil { if !utils.IsHTTPErrorCode(err, http.StatusNotFound) { return err } } return nil }
// Delete deletes the Backend for the given port. func (b *Backends) Delete(port int64) (err error) { name := b.namer.BeName(port) glog.Infof("Deleting backend %v", name) defer func() { if utils.IsHTTPErrorCode(err, http.StatusNotFound) { err = nil } if err == nil { b.snapshotter.Delete(portKey(port)) } }() // Try deleting health checks even if a backend is not found. if err = b.cloud.DeleteBackendService(name); err != nil && !utils.IsHTTPErrorCode(err, http.StatusNotFound) { return err } if err = b.healthChecker.Delete(port); err != nil && !utils.IsHTTPErrorCode(err, http.StatusNotFound) { return err } return nil }
// IsHealthy returns an error if the cluster manager is unhealthy. func (c *ClusterManager) IsHealthy() (err error) { // TODO: Expand on this, for now we just want to detect when the GCE client // is broken. _, err = c.backendPool.List() // If this container is scheduled on a node without compute/rw it is // effectively useless, but it is healthy. Reporting it as unhealthy // will lead to container crashlooping. if utils.IsHTTPErrorCode(err, http.StatusForbidden) { glog.Infof("Reporting cluster as healthy, but unable to list backends: %v", err) return nil } return }
// Sync syncs kubernetes instances with the instances in the instance group. func (i *Instances) Sync(nodes []string) (err error) { glog.V(3).Infof("Syncing nodes %v", nodes) defer func() { // The node pool is only responsible for syncing nodes to instance // groups. It never creates/deletes, so if an instance groups is // not found there's nothing it can do about it anyway. Most cases // this will happen because the backend pool has deleted the instance // group, however if it happens because a user deletes the IG by mistake // we should just wait till the backend pool fixes it. if utils.IsHTTPErrorCode(err, http.StatusNotFound) { glog.Infof("Node pool encountered a 404, ignoring: %v", err) err = nil } }() pool := i.snapshotter.Snapshot() for name := range pool { gceNodes := sets.NewString() gceNodes, err = i.list(name) if err != nil { return err } kubeNodes := sets.NewString(nodes...) // A node deleted via kubernetes could still exist as a gce vm. We don't // want to route requests to it. Similarly, a node added to kubernetes // needs to get added to the instance group so we do route requests to it. removeNodes := gceNodes.Difference(kubeNodes).List() addNodes := kubeNodes.Difference(gceNodes).List() if len(removeNodes) != 0 { if err = i.Remove( name, gceNodes.Difference(kubeNodes).List()); err != nil { return err } } if len(addNodes) != 0 { if err = i.Add( name, kubeNodes.Difference(gceNodes).List()); err != nil { return err } } } return nil }
// Cleanup deletes resources specific to this l7 in the right order. // forwarding rule -> target proxy -> url map // This leaves backends and health checks, which are shared across loadbalancers. func (l *L7) Cleanup() error { if l.fw != nil { glog.Infof("Deleting global forwarding rule %v", l.fw.Name) if err := l.cloud.DeleteGlobalForwardingRule(l.fw.Name); err != nil { if !utils.IsHTTPErrorCode(err, http.StatusNotFound) { return err } } l.fw = nil } if l.fws != nil { glog.Infof("Deleting global forwarding rule %v", l.fws.Name) if err := l.cloud.DeleteGlobalForwardingRule(l.fws.Name); err != nil { if !utils.IsHTTPErrorCode(err, http.StatusNotFound) { return err } l.fws = nil } } if l.ip != nil { glog.Infof("Deleting static IP %v(%v)", l.ip.Name, l.ip.Address) if err := l.cloud.DeleteGlobalStaticIP(l.ip.Name); err != nil { if !utils.IsHTTPErrorCode(err, http.StatusNotFound) { return err } l.ip = nil } } if l.tps != nil { glog.Infof("Deleting target https proxy %v", l.tps.Name) if err := l.cloud.DeleteTargetHttpsProxy(l.tps.Name); err != nil { if !utils.IsHTTPErrorCode(err, http.StatusNotFound) { return err } } l.tps = nil } if l.sslCert != nil { glog.Infof("Deleting sslcert %v", l.sslCert.Name) if err := l.cloud.DeleteSslCertificate(l.sslCert.Name); err != nil { if !utils.IsHTTPErrorCode(err, http.StatusNotFound) { return err } } l.sslCert = nil } if l.tp != nil { glog.Infof("Deleting target http proxy %v", l.tp.Name) if err := l.cloud.DeleteTargetHttpProxy(l.tp.Name); err != nil { if !utils.IsHTTPErrorCode(err, http.StatusNotFound) { return err } } l.tp = nil } if l.um != nil { glog.Infof("Deleting url map %v", l.um.Name) if err := l.cloud.DeleteUrlMap(l.um.Name); err != nil { if !utils.IsHTTPErrorCode(err, http.StatusNotFound) { return err } } l.um = nil } return nil }
// sync manages Ingress create/updates/deletes. func (lbc *LoadBalancerController) sync(key string) (err error) { if !lbc.hasSynced() { time.Sleep(storeSyncPollPeriod) return fmt.Errorf("Waiting for stores to sync") } glog.V(3).Infof("Syncing %v", key) paths, err := lbc.ingLister.List() if err != nil { return err } nodePorts := lbc.tr.toNodePorts(&paths) lbNames := lbc.ingLister.Store.ListKeys() lbs, err := lbc.ListRuntimeInfo() if err != nil { return err } nodeNames, err := lbc.getReadyNodeNames() if err != nil { return err } obj, ingExists, err := lbc.ingLister.Store.GetByKey(key) if err != nil { return err } // This performs a 2 phase checkpoint with the cloud: // * Phase 1 creates/verifies resources are as expected. At the end of a // successful checkpoint we know that existing L7s are WAI, and the L7 // for the Ingress associated with "key" is ready for a UrlMap update. // If this encounters an error, eg for quota reasons, we want to invoke // Phase 2 right away and retry checkpointing. // * Phase 2 performs GC by refcounting shared resources. This needs to // happen periodically whether or not stage 1 fails. At the end of a // successful GC we know that there are no dangling cloud resources that // don't have an associated Kubernetes Ingress/Service/Endpoint. defer func() { if deferErr := lbc.CloudClusterManager.GC(lbNames, nodePorts); deferErr != nil { err = fmt.Errorf("Error during sync %v, error during GC %v", err, deferErr) } glog.V(3).Infof("Finished syncing %v", key) }() // Record any errors during sync and throw a single error at the end. This // allows us to free up associated cloud resources ASAP. var syncError error if err := lbc.CloudClusterManager.Checkpoint(lbs, nodeNames, nodePorts); err != nil { // TODO: Implement proper backoff for the queue. eventMsg := "GCE" if utils.IsHTTPErrorCode(err, http.StatusForbidden) { eventMsg += " :Quota" } if ingExists { lbc.recorder.Eventf(obj.(*extensions.Ingress), api.EventTypeWarning, eventMsg, err.Error()) } else { err = fmt.Errorf("%v Error: %v", eventMsg, err) } syncError = err } if !ingExists { return syncError } // Update the UrlMap of the single loadbalancer that came through the watch. l7, err := lbc.CloudClusterManager.l7Pool.Get(key) if err != nil { return fmt.Errorf("%v, unable to get loadbalancer: %v", syncError, err) } ing := *obj.(*extensions.Ingress) if urlMap, err := lbc.tr.toUrlMap(&ing); err != nil { syncError = fmt.Errorf("%v, convert to url map error %v", syncError, err) } else if err := l7.UpdateUrlMap(urlMap); err != nil { lbc.recorder.Eventf(&ing, api.EventTypeWarning, "UrlMap", err.Error()) syncError = fmt.Errorf("%v, update url map error: %v", syncError, err) } else if err := lbc.updateIngressStatus(l7, ing); err != nil { lbc.recorder.Eventf(&ing, api.EventTypeWarning, "Status", err.Error()) syncError = fmt.Errorf("%v, update ingress error: %v", syncError, err) } return syncError }