Пример #1
0
// watchPods watches updates from kubernetes and updates the cluster information (and kicks onos) when membership
// changes.
func watchPods(kube string) {

	cluster := onos.StringSet{}
	// The set in the cluster will always include myself.
	ip, err := onos.GetMyIP()
	if err != nil {
		// add loopback, may not be the best solution
		cluster.Add("127.0.0.1")
	} else {
		cluster.Add(ip)
	}

	// We are going to use a SSL transport with verification turned off
	timeout, err := time.ParseDuration(httpTimeout)
	if err != nil {
		log.Printf("ERROR: unable to parse default HTTP timeout of '%s', will default to no timeout\n", httpTimeout)
	}
	tr := &http.Transport{
		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
		TLSHandshakeTimeout:   timeout,
		ResponseHeaderTimeout: timeout,
	}
	client := &http.Client{
		Transport: tr,
		Timeout:   timeout,
	}

	log.Printf("INFO: fetch cluster information from 'https://%s@%s/api/v1/namespaces/default/pods?labelSelector=%s'\n",
		kubeCreds, kube, url.QueryEscape(kubeOnosSelector))

	resp, err := client.Get("https://" + kubeCreds + "@" + kube + "/api/v1/namespaces/default/pods?labelSelector=" + url.QueryEscape(kubeOnosSelector))
	if err != nil {
		log.Fatalf("ERROR: Unable to communciate to kubernetes to maintain cluster information: %s\n", err)
	}

	var data map[string]interface{}
	err = json.NewDecoder(resp.Body).Decode(&data)
	if err != nil {
		log.Fatalf("ERROR: Unable to parse response back from kubernetes: %s\n", err)
	}

	// Populate the cluster set with the base from the query
	jq := jsonq.NewQuery(data)
	items, err := jq.Array("items")
	if err != nil {
		log.Printf("ERROR: Unexpected response from kubernetes: %s\n", err)
	} else {
		modified := false
		for _, item := range items {
			jq = jsonq.NewQuery(item)
			ip, err = jq.String("status.podIP")
			if err == nil {
				if !cluster.Contains(ip) {
					cluster.Add(ip)
					modified = true
				}
			}
		}
		if modified {
			onos.WriteClusterConfig(cluster.Array())
		} else {
			log.Println("INFO: no modification of cluster information based on update from kubernetes")
		}
	}

	log.Printf("INFO: base set of cluster members is %v\n", cluster.Array())

	b, _ := json.MarshalIndent(data, "", "    ")
	log.Printf("DEBUG: %s\n", string(b))

	errCount := 0
	client.Timeout = 0
	for {
		resp, err = client.Get("https://" + kubeCreds + "@" + kube + "/api/v1/namespaces/default/pods?labelSelector=" + url.QueryEscape(kubeOnosSelector) + "&watch=true")
		if err != nil {
			errCount++
			if errCount > maxErrorCount {
				log.Fatalf("ERROR: Too many errors (%d) while attempting to communicate with kubernetes: %s", errCount, err)
			}
		} else {
			// Worked, reset error count
			errCount = 0

			decoder := json.NewDecoder(resp.Body)
			if err != nil {
				errCount++
				if errCount > maxErrorCount {
					log.Fatalf("ERROR: Too many errors (%d) while attempting to communicate with kubernetes: %s", errCount, err)
				}
			} else {
				// Worked, reset error count
				errCount = 0

				for {
					var data map[string]interface{}
					err := decoder.Decode(&data)
					if err == nil {
						b, _ := json.MarshalIndent(data, "", "    ")
						log.Printf("DEBUG: retrieved: %v\n", string(b))
						jq := jsonq.NewQuery(data)
						ip, err = jq.String("object.status.podIP")
						if err == nil {
							modified := false
							log.Printf("IP: (%s) %s == %s\n", jq.AsString("type"), jq.AsString("object.metadata.name"),
								jq.AsString("object.status.podIP"))
							switch jq.AsString("type") {
							case "DELETED":
								if cluster.Contains(ip) {
									cluster.Remove(ip)
									modified = true
								}
							case "MODIFIED":
								fallthrough
							case "ADDED":
								if !cluster.Contains(ip) {
									cluster.Add(ip)
									modified = true
								}
							}
							if modified {
								onos.WriteClusterConfig(cluster.Array())
							} else {
								log.Println("INFO: no modification of cluster information based on update from kubernetes")
							}
						} else {
							log.Printf("ERROR COULD NOT FIND IP: %s\n", err)
						}
					} else {
						log.Printf("ERROR: unable to decode %s\n", err)
					}
				}
			}
		}
	}
}
Пример #2
0
// watchPods watches updates from kubernetes and updates the cluster information (and kicks onos) when membership
// changes.
func watchPods(kube string) {

	cluster := onosms.StringSet{}
	byName := map[string]string{}

	// The set in the cluster will always include myself.
	ip, err := onosms.GetMyIP()
	if err != nil {
		// add loopback, may not be the best solution
		cluster.Add("127.0.0.1")
	} else {
		cluster.Add(ip)
	}

	// We are going to use a SSL transport with verification turned off
	timeout, err := time.ParseDuration(httpTimeout)
	if err != nil {
		log.Printf("ERROR: unable to parse default HTTP timeout of '%s', will default to no timeout\n", httpTimeout)
	}

	bearer := ""
	if b, err := ioutil.ReadFile(serviceAccountTokenFile); err == nil {
		log.Println("DEBUG: successfully read authentication bearer from secrets")
		bearer = string(b)
	} else {
		log.Printf("DEBUG: unable to read token file, '%s': %s\n", serviceAccountTokenFile, err)
	}

	var caCertPool *x509.CertPool
	var cacert []byte
	if cacert, err = ioutil.ReadFile(serviceAccountCACertFile); err == nil {
		log.Println("DEBUG: successfully read authentication ca certificate")

		caCertPool = x509.NewCertPool()
		if ok := caCertPool.AppendCertsFromPEM(cacert); !ok {
			log.Fatalln("Unable to load cert from file")
		} else {
			log.Printf("DEBUG: add certificate to pool: %s\n", string(cacert))
		}
	} else {
		log.Printf("DEBUG: unable to read ca certificate file, '%s': %s\n", serviceAccountCACertFile, err)
	}

	// If we have a bearer and a cert then we should be using those as credentials, if not we will
	// check the environment
	kubeCreds := ""
	if bearer != "" && caCertPool != nil {
		kubeUser := os.Getenv(kubeUserKey)
		kubePassword := os.Getenv(kubePasswordKey)
		log.Printf("DEBUG: ENVIRONMENT '%s' '%s'\n", kubeUser, kubePassword)
		if kubeUser != "" {
			val, err := base64.StdEncoding.DecodeString(kubeUser)
			if err == nil {
				kubeUser = string(val)
				val, err = base64.StdEncoding.DecodeString(kubePassword)
				if err == nil {
					kubePassword = string(val)
					kubeCreds = kubeUser + ":" + kubePassword + "@"
				} else {
					log.Printf("ERROR: unable to decode password (%s) for kubernetes api: %s\n", kubeUser, err)
				}
			} else {
				log.Printf("ERROR: unable to decode username (%s) for kubernetes api: %s\n", kubePassword, err)
			}
		}
	}

	log.Printf("INFO: fetch cluster information from 'https://%s%s/api/v1/namespaces/default/pods?labelSelector=%s'\n",
		kubeCreds, kube, url.QueryEscape(kubeOnosSelector))

	req, err := http.NewRequest("GET", "https://"+kubeCreds+kube+"/api/v1/namespaces/default/pods?labelSelector="+url.QueryEscape(kubeOnosSelector), nil)
	if err != nil {
		log.Fatalf("ERROR: Unable to build http request to kubernetes API server: %s\n", err)
	}
	if len(bearer) > 0 {
		log.Printf("DEBUG: adding to header: %s %s\n", httpBearerHeader, bearer)
		req.Header.Add(httpAuthorizationHeader, httpBearerHeader+" "+bearer)
	}

	client := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
				RootCAs:            caCertPool,
			},
			TLSHandshakeTimeout:   timeout,
			ResponseHeaderTimeout: timeout,
		},
		Timeout: timeout,
	}

	resp, err := client.Do(req)
	if err != nil {
		log.Fatalf("ERROR: Unable to communciate to kubernetes to maintain cluster information: %s\n", err)
	}
	log.Printf("DEBUG: response back from kubernetes: %s\n", resp.Status)

	if int(resp.StatusCode/100) != 2 {
		log.Fatalf("ERROR: bad response code back from kubernetes: %s\n", resp.Status)
	}

	var data map[string]interface{}
	err = json.NewDecoder(resp.Body).Decode(&data)
	if err != nil {
		log.Fatalf("ERROR: Unable to parse response back from kubernetes: %s\n", err)
	}

	// Populate the cluster set with the base from the query
	jq := jsonq.NewQuery(data)
	items, err := jq.Array("items")
	if err != nil {
		log.Printf("ERROR: Unexpected response from kubernetes: %s\n", err)
	} else {
		modified := false
		for _, item := range items {
			jq = jsonq.NewQuery(item)
			ip, err = jq.String("status.podIP")
			if err == nil {
				if !cluster.Contains(ip) {
					cluster.Add(ip)
					modified = true
				}
				byName[jq.AsString("metadata.name")] = ip
			}
		}
		if modified {
			onosms.WriteClusterConfig(cluster.Array())
		} else {
			log.Println("INFO: no modification of cluster information based on update from kubernetes")
		}
	}

	log.Printf("INFO: base set of cluster members is %v\n", cluster.Array())

	b, _ := json.MarshalIndent(data, "", "    ")
	log.Printf("DEBUG: %s\n", string(b))

	errCount := 0
	client.Timeout = 0
	for {
		req, err := http.NewRequest("GET",
			"https://"+kubeCreds+kube+"/api/v1/namespaces/default/pods?labelSelector="+url.QueryEscape(kubeOnosSelector)+"&watch=true", nil)
		if err != nil {
			errCount++
			if errCount > maxErrorCount {
				log.Fatalf("ERROR: Too many errors (%d) while attempting to build request to kubernetes: %s", errCount, err)
			}
		}
		if bearer != "" {
			log.Printf("DEBUG: adding to header: %s %s\n", httpBearerHeader, bearer)
			req.Header.Add(httpAuthorizationHeader, httpBearerHeader+" "+bearer)
		}

		resp, err := client.Do(req)
		if err != nil {
			errCount++
			if errCount > maxErrorCount {
				log.Fatalf("ERROR: Too many errors (%d) while attempting to communicate with kubernetes: %s", errCount, err)
			}
		} else {
			// Worked, reset error count
			errCount = 0

			decoder := json.NewDecoder(resp.Body)
			if err != nil {
				errCount++
				if errCount > maxErrorCount {
					log.Fatalf("ERROR: Too many errors (%d) while attempting to communicate with kubernetes: %s", errCount, err)
				}
			} else {
				// Worked, reset error count
				errCount = 0

				for {
					var data map[string]interface{}
					err := decoder.Decode(&data)
					if err == nil {
						log.Printf("DEBUG: cluster = %v\n", cluster)
						log.Printf("DEUBG: byName = %v\n", byName)
						b, _ := json.MarshalIndent(data, "", "    ")
						log.Printf("DEBUG: retrieved: %v\n", string(b))
						jq := jsonq.NewQuery(data)
						name := jq.AsString("object.metadata.name")
						ip, err = jq.String("object.status.podIP")
						modified := false
						log.Printf("IP: (%s) %s == %s | %s\n", jq.AsString("type"), name, ip, byName[name])
						switch jq.AsString("type") {
						case "DELETED":
							if ip == "" {
								ip = byName[name]
							}
							if ip != "" {
								if cluster.Contains(ip) {
									cluster.Remove(ip)
									modified = true
								}
								delete(byName, name)
							} else {
								log.Printf("ERROR: Unable to determine podIP for pod being deleted: %s\n", err)
							}
						case "MODIFIED":
							fallthrough
						case "ADDED":
							if ip != "" {
								if !cluster.Contains(ip) {
									cluster.Add(ip)
									modified = true
								}
								byName[name] = ip
							} else {
								log.Println("INFO: Update without a podIP")
							}
						}
						if modified {
							onosms.WriteClusterConfig(cluster.Array())
						} else {
							log.Println("INFO: no modification of cluster information based on update from kubernetes")
						}

					} else {
						log.Printf("ERROR: unable to decode %s\n", err)
					}
				}
			}
		}
	}
}