// Read state from key. func (d *ConsulStateDriver) Read(key string) ([]byte, error) { key = processKey(key) kv, _, err := d.Client.KV().Get(key, nil) if err != nil { if api.IsServerError(err) || strings.Contains(err.Error(), "EOF") || strings.Contains(err.Error(), "connection refused") { for i := 0; i < maxConsulRetries; i++ { kv, _, err = d.Client.KV().Get(key, nil) if err == nil { break } // Retry after a delay time.Sleep(time.Second) } } else { return []byte{}, err } } // Consul returns success and a nil kv when a key is not found, // translate it to 'Key not found' error if kv == nil { return []byte{}, core.Errorf("Key not found") } return kv.Value, err }
// WatchAll state transitions from baseKey func (d *ConsulStateDriver) WatchAll(baseKey string, rsps chan [2][]byte) error { baseKey = processKey(baseKey) consulRsps := make(chan api.KVPairs, 1) stop := make(chan bool, 1) recvErr := make(chan error, 2) // Consul returns all the keys as return value of List(). The following maps helps // track the state that has been seen and used to appropriately generate // create, modify and delete events kvCache := map[string]*api.KVPair{} // read with index=0 to fetch all existing keys var waitIndex uint64 kvs, qm, err := d.Client.KV().List(baseKey, &api.QueryOptions{WaitIndex: waitIndex}) if err != nil { log.Errorf("consul read failed for key %q. Error: %s", baseKey, err) return err } // Consul returns success and a nil kv when a key is not found. // Treat this as starting with no state. // XXX: shall we fail the watch in this case? if kvs == nil { kvs = api.KVPairs{} } for _, kv := range kvs { kvCache[kv.Key] = kv } waitIndex = qm.LastIndex go d.channelConsulEvents(baseKey, kvCache, consulRsps, rsps, recvErr, stop) for { select { case err := <-recvErr: return err default: kvs, qm, err := d.Client.KV().List(baseKey, &api.QueryOptions{WaitIndex: waitIndex}) if err != nil { if api.IsServerError(err) || strings.Contains(err.Error(), "EOF") || strings.Contains(err.Error(), "connection refused") { log.Warnf("Consul watch: server error: %v for %s. Retrying..", err, baseKey) time.Sleep(5 * time.Second) continue } else { log.Errorf("consul watch failed for key %q. Error: %s. stopping watch..", baseKey, err) stop <- true return err } } // Consul returns success and a nil kv when a key is not found. // This shall translate into appropriate 'Delete' events or // no events (depending on whether some keys were seen before) // XXX: shall we stop the watch in this case? if kvs == nil { kvs = api.KVPairs{} } waitIndex = qm.LastIndex consulRsps <- kvs } } }
// SetObj writes an object func (cp *ConsulClient) SetObj(key string, value interface{}) error { key = processKey("/contiv.io/obj/" + processKey(key)) // JSON format the object jsonVal, err := json.Marshal(value) if err != nil { log.Errorf("Json conversion error. Err %v", err) return err } _, err = cp.client.KV().Put(&api.KVPair{Key: key, Value: jsonVal}, nil) if err != nil { if api.IsServerError(err) || strings.Contains(err.Error(), "EOF") || strings.Contains(err.Error(), "connection refused") { for i := 0; i < maxConsulRetries; i++ { _, err = cp.client.KV().Put(&api.KVPair{Key: key, Value: jsonVal}, nil) if err == nil { break } // Retry after a delay time.Sleep(time.Second) } } } return err }
func inputFile(kv *consulapi.KV, path string) { f, err := os.Open(path) if err != nil { log.Fatal(err.Error()) } r := csv.NewReader(f) for { record, err := r.Read() if err == io.EOF { break } if err != nil { log.Fatal(err) } if len(record) != 2 { fmt.Printf("Incorrectly formatted line: %v\n", record) } d := &consulapi.KVPair{Key: record[0], Value: []byte(record[1])} _, err = kv.Put(d, nil) if err != nil { if consulapi.IsServerError(err) && inRetries < 10 { log.Println("consul server err: retry after 1s") <-time.After(time.Second) inRetries++ inputFile(kv, path) return } log.Fatal(err.Error()) } } }
func monitorConsulGoroutine(kv *consulapi.KV, service string, lastIndex uint64) { for { pair, qm, err := kv.Get( consulPrefix+service, &consulapi.QueryOptions{ WaitIndex: lastIndex, RequireConsistent: true, }) if err != nil { if consulapi.IsServerError(err) { // Consul unavailable. Try again. time.Sleep(1 * time.Second) continue } if strings.Contains(err.Error(), "read: connection timed out") { // Try again. time.Sleep(1 * time.Second) continue } log.Fatalf("Error monitoring config in Consul: %v\n", err) } if pair == nil { log.Fatalf("Config in consul has been deleted\n") } updateConsulFlags(pair.Value) lastIndex = qm.LastIndex } }
// WaitResource monitors a resource and blocks until that resource is // released or there is some other error. func WaitResource(service string, resource string) error { service = url.QueryEscape(service) resource = url.QueryEscape(resource) consul := config.GetConsulClient() kv := consul.KV() lastIndex := uint64(0) for { pair, qm, err := kv.Get( consulResourcePrefix+service+"/"+resource, &consulapi.QueryOptions{ WaitIndex: lastIndex, RequireConsistent: true, }) if err != nil { if !consulapi.IsServerError(err) { return err } // Consul unresponsive. Wait a bit and try again. time.Sleep(3 * time.Second) continue } if pair == nil { return nil } lastIndex = qm.LastIndex } }
// Init initializes the consul client func (cp *consulPlugin) NewClient(endpoints []string) (API, error) { cc := new(ConsulClient) if len(endpoints) == 0 { endpoints = []string{"127.0.0.1:8500"} } // default consul config cc.consulConfig = api.Config{Address: strings.TrimPrefix(endpoints[0], "http://")} // Initialize service DB cc.serviceDb = make(map[string]*consulServiceState) // Init consul client client, err := api.NewClient(&cc.consulConfig) if err != nil { log.Fatalf("Error initializing consul client") return nil, err } cc.client = client // verify we can reach the consul _, _, err = client.KV().List("/", nil) if err != nil { if api.IsServerError(err) || strings.Contains(err.Error(), "EOF") || strings.Contains(err.Error(), "connection refused") { for i := 0; i < maxConsulRetries; i++ { _, _, err = client.KV().List("/", nil) if err == nil { break } // Retry after a delay time.Sleep(time.Second) } } // return error if it failed after retries if err != nil { log.Errorf("Error connecting to consul. Err: %v", err) return nil, err } } return cc, nil }
func monitorConsul() { service := ConfigFlag.Get() if service == "" { return } client := GetConsulClient() kv := client.KV() var ( pair *consulapi.KVPair qm *consulapi.QueryMeta err error ) for retry := 0; retry < 20; retry++ { pair, qm, err = kv.Get( consulPrefix+service, &consulapi.QueryOptions{ RequireConsistent: true, }) if err != nil { if consulapi.IsServerError(err) { // Consul unavailable. Try again. time.Sleep(1 * time.Second) continue } else { break } } if pair == nil { // Config does not exist. Maybe not yet uploaded. Try again. err = fmt.Errorf("Config not found") time.Sleep(1 * time.Second) continue } break } if err != nil { log.Fatalf( "Error trying to get config for the first time from consul: %v\n", err) } updateConsulFlags(pair.Value) go monitorConsulGoroutine(kv, service, qm.LastIndex) }
// Write state to key with value. func (d *ConsulStateDriver) Write(key string, value []byte) error { key = processKey(key) _, err := d.Client.KV().Put(&api.KVPair{Key: key, Value: value}, nil) if err != nil && (api.IsServerError(err) || strings.Contains(err.Error(), "EOF") || strings.Contains(err.Error(), "connection refused")) { for i := 0; i < maxConsulRetries; i++ { _, err = d.Client.KV().Put(&api.KVPair{Key: key, Value: value}, nil) if err == nil { break } // Retry after a delay time.Sleep(time.Second) } } return err }
// DelObj deletes an object func (cp *ConsulClient) DelObj(key string) error { key = processKey("/contiv.io/obj/" + processKey(key)) _, err := cp.client.KV().Delete(key, nil) if err != nil { if api.IsServerError(err) || strings.Contains(err.Error(), "EOF") || strings.Contains(err.Error(), "connection refused") { for i := 0; i < maxConsulRetries; i++ { _, err = cp.client.KV().Delete(key, nil) if err == nil { break } // Retry after a delay time.Sleep(time.Second) } } } return err }
// GetObj reads the object func (cp *ConsulClient) GetObj(key string, retVal interface{}) error { key = processKey("/contiv.io/obj/" + processKey(key)) resp, _, err := cp.client.KV().Get(key, &api.QueryOptions{RequireConsistent: true}) if err != nil { if api.IsServerError(err) || strings.Contains(err.Error(), "EOF") || strings.Contains(err.Error(), "connection refused") { for i := 0; i < maxConsulRetries; i++ { resp, _, err = cp.client.KV().Get(key, &api.QueryOptions{RequireConsistent: true}) if err == nil { break } // Retry after a delay time.Sleep(time.Second) } } // return error if it failed after retries if err != nil { return err } } // Consul returns success and a nil kv when a key is not found, // translate it to 'Key not found' error if resp == nil { return errors.New("Key not found") } // Parse JSON response if err := json.Unmarshal(resp.Value, retVal); err != nil { log.Errorf("Error parsing object %v, Err %v", resp.Value, err) return err } return nil }
func outputFile(kv *consulapi.KV, path, prefix string) { vs, _, err := kv.List(prefix, nil) if err != nil { if consulapi.IsServerError(err) && outRetries < 10 { log.Println("consul server err: retry after 1s") <-time.After(time.Second) outRetries++ outputFile(kv, path, prefix) return } log.Fatal(err.Error()) } f, err := os.Create(path) if err != nil { log.Fatal(err.Error()) } defer f.Close() for _, val := range vs { fmt.Fprintf(f, "%s,%s\n", val.Key, val.Value) } f.Sync() }
// ListDir returns a list of keys in a directory func (cp *ConsulClient) ListDir(key string) ([]string, error) { key = processKey("/contiv.io/obj/" + processKey(key)) kvs, _, err := cp.client.KV().List(key, nil) if err != nil { if api.IsServerError(err) || strings.Contains(err.Error(), "EOF") || strings.Contains(err.Error(), "connection refused") { for i := 0; i < maxConsulRetries; i++ { kvs, _, err = cp.client.KV().List(key, nil) if err == nil { break } // Retry after a delay time.Sleep(time.Second) } } // return error if it failed after retries if err != nil { return nil, err } } // Consul returns success and a nil kv when a key is not found, // translate it to 'Key not found' error if kvs == nil { return []string{}, nil } var keys []string for _, kv := range kvs { keys = append(keys, string(kv.Value)) } return keys, nil }
// WatchService watches for service instance changes func (cp *ConsulClient) WatchService(srvName string, eventCh chan WatchServiceEvent, stopCh chan bool) error { keyName := "contiv.io/service/" + srvName + "/" // Run in background go func() { var currSrvMap = make(map[string]ServiceInfo) // Get current list of services srvList, lastIdx, err := cp.getServiceInstances(keyName, 0) if err != nil { log.Errorf("Error getting service instances for (%s): Err: %v", srvName, err) } else { // for each instance trigger an add event for _, srvInfo := range srvList { eventCh <- WatchServiceEvent{ EventType: WatchServiceEventAdd, ServiceInfo: srvInfo, } // Add the service to local cache srvKey := srvInfo.HostAddr + ":" + strconv.Itoa(srvInfo.Port) currSrvMap[srvKey] = srvInfo } } // Loop till asked to stop for { // Check if we should quit select { case <-stopCh: return default: // Read the service instances srvList, lastIdx, err = cp.getServiceInstances(keyName, lastIdx) if err != nil { if api.IsServerError(err) || strings.Contains(err.Error(), "EOF") || strings.Contains(err.Error(), "connection refused") { log.Warnf("Consul service watch: server error: %v Retrying..", err) } else { log.Errorf("Error getting service instances for (%s): Err: %v. Exiting watch", srvName, err) } // Wait a little and continue time.Sleep(5 * time.Second) continue } else { log.Debugf("Got consul srv list: {%+v}. Curr: {%+v}", srvList, currSrvMap) var newSrvMap = make(map[string]ServiceInfo) // Check if there are any new services for _, srvInfo := range srvList { srvKey := srvInfo.HostAddr + ":" + strconv.Itoa(srvInfo.Port) // If the entry didnt exists previously, trigger add event if _, ok := currSrvMap[srvKey]; !ok { log.Debugf("Sending add event for srv: %v", srvInfo) eventCh <- WatchServiceEvent{ EventType: WatchServiceEventAdd, ServiceInfo: srvInfo, } } // create new service map newSrvMap[srvKey] = srvInfo } // for all entries in old service list, see if we need to delete any for _, srvInfo := range currSrvMap { srvKey := srvInfo.HostAddr + ":" + strconv.Itoa(srvInfo.Port) // if the entry does not exists in new list, delete it if _, ok := newSrvMap[srvKey]; !ok { log.Debugf("Sending delete event for srv: %v", srvInfo) eventCh <- WatchServiceEvent{ EventType: WatchServiceEventDel, ServiceInfo: srvInfo, } } } // set new srv map as the current currSrvMap = newSrvMap } } } }() return nil }