// watchPods watches updates from kubernetes and updates the cluster information (and kicks onos) when membership
// changes.
func watchPods(kube string) {
cluster := onos.StringSet{}
// The set in the cluster will always include myself.
ip, err := onos.GetMyIP()
if err != nil {
// add loopback, may not be the best solution
cluster.Add("127.0.0.1")
} else {
cluster.Add(ip)
}
// We are going to use a SSL transport with verification turned off
timeout, err := time.ParseDuration(httpTimeout)
if err != nil {
log.Printf("ERROR: unable to parse default HTTP timeout of '%s', will default to no timeout\n", httpTimeout)
}
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
TLSHandshakeTimeout: timeout,
ResponseHeaderTimeout: timeout,
}
client := &http.Client{
Transport: tr,
Timeout: timeout,
}
log.Printf("INFO: fetch cluster information from 'https://%s@%s/api/v1/namespaces/default/pods?labelSelector=%s'\n",
kubeCreds, kube, url.QueryEscape(kubeOnosSelector))
resp, err := client.Get("https://" + kubeCreds + "@" + kube + "/api/v1/namespaces/default/pods?labelSelector=" + url.QueryEscape(kubeOnosSelector))
if err != nil {
log.Fatalf("ERROR: Unable to communciate to kubernetes to maintain cluster information: %s\n", err)
}
var data map[string]interface{}
err = json.NewDecoder(resp.Body).Decode(&data)
if err != nil {
log.Fatalf("ERROR: Unable to parse response back from kubernetes: %s\n", err)
}
// Populate the cluster set with the base from the query
jq := jsonq.NewQuery(data)
items, err := jq.Array("items")
if err != nil {
log.Printf("ERROR: Unexpected response from kubernetes: %s\n", err)
} else {
modified := false
for _, item := range items {
jq = jsonq.NewQuery(item)
ip, err = jq.String("status.podIP")
if err == nil {
if !cluster.Contains(ip) {
cluster.Add(ip)
modified = true
}
}
}
if modified {
onos.WriteClusterConfig(cluster.Array())
} else {
log.Println("INFO: no modification of cluster information based on update from kubernetes")
}
}
log.Printf("INFO: base set of cluster members is %v\n", cluster.Array())
b, _ := json.MarshalIndent(data, "", " ")
log.Printf("DEBUG: %s\n", string(b))
errCount := 0
client.Timeout = 0
for {
resp, err = client.Get("https://" + kubeCreds + "@" + kube + "/api/v1/namespaces/default/pods?labelSelector=" + url.QueryEscape(kubeOnosSelector) + "&watch=true")
if err != nil {
errCount++
if errCount > maxErrorCount {
log.Fatalf("ERROR: Too many errors (%d) while attempting to communicate with kubernetes: %s", errCount, err)
}
} else {
// Worked, reset error count
errCount = 0
decoder := json.NewDecoder(resp.Body)
if err != nil {
errCount++
if errCount > maxErrorCount {
log.Fatalf("ERROR: Too many errors (%d) while attempting to communicate with kubernetes: %s", errCount, err)
}
} else {
// Worked, reset error count
errCount = 0
for {
var data map[string]interface{}
err := decoder.Decode(&data)
if err == nil {
b, _ := json.MarshalIndent(data, "", " ")
log.Printf("DEBUG: retrieved: %v\n", string(b))
jq := jsonq.NewQuery(data)
ip, err = jq.String("object.status.podIP")
if err == nil {
modified := false
log.Printf("IP: (%s) %s == %s\n", jq.AsString("type"), jq.AsString("object.metadata.name"),
jq.AsString("object.status.podIP"))
switch jq.AsString("type") {
case "DELETED":
if cluster.Contains(ip) {
cluster.Remove(ip)
modified = true
}
case "MODIFIED":
fallthrough
case "ADDED":
if !cluster.Contains(ip) {
cluster.Add(ip)
modified = true
}
}
if modified {
onos.WriteClusterConfig(cluster.Array())
} else {
log.Println("INFO: no modification of cluster information based on update from kubernetes")
}
} else {
log.Printf("ERROR COULD NOT FIND IP: %s\n", err)
}
} else {
log.Printf("ERROR: unable to decode %s\n", err)
}
}
}
}
}
}
// watchPods watches updates from kubernetes and updates the cluster information (and kicks onos) when membership
// changes.
func watchPods(kube string) {
cluster := onosms.StringSet{}
byName := map[string]string{}
// The set in the cluster will always include myself.
ip, err := onosms.GetMyIP()
if err != nil {
// add loopback, may not be the best solution
cluster.Add("127.0.0.1")
} else {
cluster.Add(ip)
}
// We are going to use a SSL transport with verification turned off
timeout, err := time.ParseDuration(httpTimeout)
if err != nil {
log.Printf("ERROR: unable to parse default HTTP timeout of '%s', will default to no timeout\n", httpTimeout)
}
bearer := ""
if b, err := ioutil.ReadFile(serviceAccountTokenFile); err == nil {
log.Println("DEBUG: successfully read authentication bearer from secrets")
bearer = string(b)
} else {
log.Printf("DEBUG: unable to read token file, '%s': %s\n", serviceAccountTokenFile, err)
}
var caCertPool *x509.CertPool
var cacert []byte
if cacert, err = ioutil.ReadFile(serviceAccountCACertFile); err == nil {
log.Println("DEBUG: successfully read authentication ca certificate")
caCertPool = x509.NewCertPool()
if ok := caCertPool.AppendCertsFromPEM(cacert); !ok {
log.Fatalln("Unable to load cert from file")
} else {
log.Printf("DEBUG: add certificate to pool: %s\n", string(cacert))
}
} else {
log.Printf("DEBUG: unable to read ca certificate file, '%s': %s\n", serviceAccountCACertFile, err)
}
// If we have a bearer and a cert then we should be using those as credentials, if not we will
// check the environment
kubeCreds := ""
if bearer != "" && caCertPool != nil {
kubeUser := os.Getenv(kubeUserKey)
kubePassword := os.Getenv(kubePasswordKey)
log.Printf("DEBUG: ENVIRONMENT '%s' '%s'\n", kubeUser, kubePassword)
if kubeUser != "" {
val, err := base64.StdEncoding.DecodeString(kubeUser)
if err == nil {
kubeUser = string(val)
val, err = base64.StdEncoding.DecodeString(kubePassword)
if err == nil {
kubePassword = string(val)
kubeCreds = kubeUser + ":" + kubePassword + "@"
} else {
log.Printf("ERROR: unable to decode password (%s) for kubernetes api: %s\n", kubeUser, err)
}
} else {
log.Printf("ERROR: unable to decode username (%s) for kubernetes api: %s\n", kubePassword, err)
}
}
}
log.Printf("INFO: fetch cluster information from 'https://%s%s/api/v1/namespaces/default/pods?labelSelector=%s'\n",
kubeCreds, kube, url.QueryEscape(kubeOnosSelector))
req, err := http.NewRequest("GET", "https://"+kubeCreds+kube+"/api/v1/namespaces/default/pods?labelSelector="+url.QueryEscape(kubeOnosSelector), nil)
if err != nil {
log.Fatalf("ERROR: Unable to build http request to kubernetes API server: %s\n", err)
}
if len(bearer) > 0 {
log.Printf("DEBUG: adding to header: %s %s\n", httpBearerHeader, bearer)
req.Header.Add(httpAuthorizationHeader, httpBearerHeader+" "+bearer)
}
client := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
RootCAs: caCertPool,
},
TLSHandshakeTimeout: timeout,
ResponseHeaderTimeout: timeout,
},
Timeout: timeout,
}
resp, err := client.Do(req)
if err != nil {
log.Fatalf("ERROR: Unable to communciate to kubernetes to maintain cluster information: %s\n", err)
}
log.Printf("DEBUG: response back from kubernetes: %s\n", resp.Status)
if int(resp.StatusCode/100) != 2 {
log.Fatalf("ERROR: bad response code back from kubernetes: %s\n", resp.Status)
}
var data map[string]interface{}
err = json.NewDecoder(resp.Body).Decode(&data)
if err != nil {
log.Fatalf("ERROR: Unable to parse response back from kubernetes: %s\n", err)
}
// Populate the cluster set with the base from the query
jq := jsonq.NewQuery(data)
items, err := jq.Array("items")
if err != nil {
log.Printf("ERROR: Unexpected response from kubernetes: %s\n", err)
} else {
modified := false
for _, item := range items {
jq = jsonq.NewQuery(item)
ip, err = jq.String("status.podIP")
if err == nil {
if !cluster.Contains(ip) {
cluster.Add(ip)
modified = true
}
byName[jq.AsString("metadata.name")] = ip
}
}
if modified {
onosms.WriteClusterConfig(cluster.Array())
} else {
log.Println("INFO: no modification of cluster information based on update from kubernetes")
}
}
log.Printf("INFO: base set of cluster members is %v\n", cluster.Array())
b, _ := json.MarshalIndent(data, "", " ")
log.Printf("DEBUG: %s\n", string(b))
errCount := 0
client.Timeout = 0
for {
req, err := http.NewRequest("GET",
"https://"+kubeCreds+kube+"/api/v1/namespaces/default/pods?labelSelector="+url.QueryEscape(kubeOnosSelector)+"&watch=true", nil)
if err != nil {
errCount++
if errCount > maxErrorCount {
log.Fatalf("ERROR: Too many errors (%d) while attempting to build request to kubernetes: %s", errCount, err)
}
}
if bearer != "" {
log.Printf("DEBUG: adding to header: %s %s\n", httpBearerHeader, bearer)
req.Header.Add(httpAuthorizationHeader, httpBearerHeader+" "+bearer)
}
resp, err := client.Do(req)
if err != nil {
errCount++
if errCount > maxErrorCount {
log.Fatalf("ERROR: Too many errors (%d) while attempting to communicate with kubernetes: %s", errCount, err)
}
} else {
// Worked, reset error count
errCount = 0
decoder := json.NewDecoder(resp.Body)
if err != nil {
errCount++
if errCount > maxErrorCount {
log.Fatalf("ERROR: Too many errors (%d) while attempting to communicate with kubernetes: %s", errCount, err)
}
} else {
// Worked, reset error count
errCount = 0
for {
var data map[string]interface{}
err := decoder.Decode(&data)
if err == nil {
log.Printf("DEBUG: cluster = %v\n", cluster)
log.Printf("DEUBG: byName = %v\n", byName)
b, _ := json.MarshalIndent(data, "", " ")
log.Printf("DEBUG: retrieved: %v\n", string(b))
jq := jsonq.NewQuery(data)
name := jq.AsString("object.metadata.name")
ip, err = jq.String("object.status.podIP")
modified := false
log.Printf("IP: (%s) %s == %s | %s\n", jq.AsString("type"), name, ip, byName[name])
switch jq.AsString("type") {
case "DELETED":
if ip == "" {
ip = byName[name]
}
if ip != "" {
if cluster.Contains(ip) {
cluster.Remove(ip)
modified = true
}
delete(byName, name)
} else {
log.Printf("ERROR: Unable to determine podIP for pod being deleted: %s\n", err)
}
case "MODIFIED":
fallthrough
case "ADDED":
if ip != "" {
if !cluster.Contains(ip) {
cluster.Add(ip)
modified = true
}
byName[name] = ip
} else {
log.Println("INFO: Update without a podIP")
}
}
if modified {
onosms.WriteClusterConfig(cluster.Array())
} else {
log.Println("INFO: no modification of cluster information based on update from kubernetes")
}
} else {
log.Printf("ERROR: unable to decode %s\n", err)
}
}
}
}
}
}