// Read state from key.
func (d *ConsulStateDriver) Read(key string) ([]byte, error) {
key = processKey(key)
kv, _, err := d.Client.KV().Get(key, nil)
if err != nil {
if api.IsServerError(err) || strings.Contains(err.Error(), "EOF") ||
strings.Contains(err.Error(), "connection refused") {
for i := 0; i < maxConsulRetries; i++ {
kv, _, err = d.Client.KV().Get(key, nil)
if err == nil {
break
}
// Retry after a delay
time.Sleep(time.Second)
}
} else {
return []byte{}, err
}
}
// Consul returns success and a nil kv when a key is not found,
// translate it to 'Key not found' error
if kv == nil {
return []byte{}, core.Errorf("Key not found")
}
return kv.Value, err
}
// WatchAll state transitions from baseKey
func (d *ConsulStateDriver) WatchAll(baseKey string, rsps chan [2][]byte) error {
baseKey = processKey(baseKey)
consulRsps := make(chan api.KVPairs, 1)
stop := make(chan bool, 1)
recvErr := make(chan error, 2)
// Consul returns all the keys as return value of List(). The following maps helps
// track the state that has been seen and used to appropriately generate
// create, modify and delete events
kvCache := map[string]*api.KVPair{}
// read with index=0 to fetch all existing keys
var waitIndex uint64
kvs, qm, err := d.Client.KV().List(baseKey, &api.QueryOptions{WaitIndex: waitIndex})
if err != nil {
log.Errorf("consul read failed for key %q. Error: %s", baseKey, err)
return err
}
// Consul returns success and a nil kv when a key is not found.
// Treat this as starting with no state.
// XXX: shall we fail the watch in this case?
if kvs == nil {
kvs = api.KVPairs{}
}
for _, kv := range kvs {
kvCache[kv.Key] = kv
}
waitIndex = qm.LastIndex
go d.channelConsulEvents(baseKey, kvCache, consulRsps, rsps, recvErr, stop)
for {
select {
case err := <-recvErr:
return err
default:
kvs, qm, err := d.Client.KV().List(baseKey, &api.QueryOptions{WaitIndex: waitIndex})
if err != nil {
if api.IsServerError(err) || strings.Contains(err.Error(), "EOF") || strings.Contains(err.Error(), "connection refused") {
log.Warnf("Consul watch: server error: %v for %s. Retrying..", err, baseKey)
time.Sleep(5 * time.Second)
continue
} else {
log.Errorf("consul watch failed for key %q. Error: %s. stopping watch..", baseKey, err)
stop <- true
return err
}
}
// Consul returns success and a nil kv when a key is not found.
// This shall translate into appropriate 'Delete' events or
// no events (depending on whether some keys were seen before)
// XXX: shall we stop the watch in this case?
if kvs == nil {
kvs = api.KVPairs{}
}
waitIndex = qm.LastIndex
consulRsps <- kvs
}
}
}
// SetObj writes an object
func (cp *ConsulClient) SetObj(key string, value interface{}) error {
key = processKey("/contiv.io/obj/" + processKey(key))
// JSON format the object
jsonVal, err := json.Marshal(value)
if err != nil {
log.Errorf("Json conversion error. Err %v", err)
return err
}
_, err = cp.client.KV().Put(&api.KVPair{Key: key, Value: jsonVal}, nil)
if err != nil {
if api.IsServerError(err) || strings.Contains(err.Error(), "EOF") ||
strings.Contains(err.Error(), "connection refused") {
for i := 0; i < maxConsulRetries; i++ {
_, err = cp.client.KV().Put(&api.KVPair{Key: key, Value: jsonVal}, nil)
if err == nil {
break
}
// Retry after a delay
time.Sleep(time.Second)
}
}
}
return err
}
func inputFile(kv *consulapi.KV, path string) {
f, err := os.Open(path)
if err != nil {
log.Fatal(err.Error())
}
r := csv.NewReader(f)
for {
record, err := r.Read()
if err == io.EOF {
break
}
if err != nil {
log.Fatal(err)
}
if len(record) != 2 {
fmt.Printf("Incorrectly formatted line: %v\n", record)
}
d := &consulapi.KVPair{Key: record[0], Value: []byte(record[1])}
_, err = kv.Put(d, nil)
if err != nil {
if consulapi.IsServerError(err) && inRetries < 10 {
log.Println("consul server err: retry after 1s")
<-time.After(time.Second)
inRetries++
inputFile(kv, path)
return
}
log.Fatal(err.Error())
}
}
}
func monitorConsulGoroutine(kv *consulapi.KV, service string, lastIndex uint64) {
for {
pair, qm, err := kv.Get(
consulPrefix+service,
&consulapi.QueryOptions{
WaitIndex: lastIndex,
RequireConsistent: true,
})
if err != nil {
if consulapi.IsServerError(err) {
// Consul unavailable. Try again.
time.Sleep(1 * time.Second)
continue
}
if strings.Contains(err.Error(), "read: connection timed out") {
// Try again.
time.Sleep(1 * time.Second)
continue
}
log.Fatalf("Error monitoring config in Consul: %v\n", err)
}
if pair == nil {
log.Fatalf("Config in consul has been deleted\n")
}
updateConsulFlags(pair.Value)
lastIndex = qm.LastIndex
}
}
// WaitResource monitors a resource and blocks until that resource is
// released or there is some other error.
func WaitResource(service string, resource string) error {
service = url.QueryEscape(service)
resource = url.QueryEscape(resource)
consul := config.GetConsulClient()
kv := consul.KV()
lastIndex := uint64(0)
for {
pair, qm, err := kv.Get(
consulResourcePrefix+service+"/"+resource,
&consulapi.QueryOptions{
WaitIndex: lastIndex,
RequireConsistent: true,
})
if err != nil {
if !consulapi.IsServerError(err) {
return err
}
// Consul unresponsive. Wait a bit and try again.
time.Sleep(3 * time.Second)
continue
}
if pair == nil {
return nil
}
lastIndex = qm.LastIndex
}
}
// Init initializes the consul client
func (cp *consulPlugin) NewClient(endpoints []string) (API, error) {
cc := new(ConsulClient)
if len(endpoints) == 0 {
endpoints = []string{"127.0.0.1:8500"}
}
// default consul config
cc.consulConfig = api.Config{Address: strings.TrimPrefix(endpoints[0], "http://")}
// Initialize service DB
cc.serviceDb = make(map[string]*consulServiceState)
// Init consul client
client, err := api.NewClient(&cc.consulConfig)
if err != nil {
log.Fatalf("Error initializing consul client")
return nil, err
}
cc.client = client
// verify we can reach the consul
_, _, err = client.KV().List("/", nil)
if err != nil {
if api.IsServerError(err) || strings.Contains(err.Error(), "EOF") ||
strings.Contains(err.Error(), "connection refused") {
for i := 0; i < maxConsulRetries; i++ {
_, _, err = client.KV().List("/", nil)
if err == nil {
break
}
// Retry after a delay
time.Sleep(time.Second)
}
}
// return error if it failed after retries
if err != nil {
log.Errorf("Error connecting to consul. Err: %v", err)
return nil, err
}
}
return cc, nil
}
func monitorConsul() {
service := ConfigFlag.Get()
if service == "" {
return
}
client := GetConsulClient()
kv := client.KV()
var (
pair *consulapi.KVPair
qm *consulapi.QueryMeta
err error
)
for retry := 0; retry < 20; retry++ {
pair, qm, err = kv.Get(
consulPrefix+service, &consulapi.QueryOptions{
RequireConsistent: true,
})
if err != nil {
if consulapi.IsServerError(err) {
// Consul unavailable. Try again.
time.Sleep(1 * time.Second)
continue
} else {
break
}
}
if pair == nil {
// Config does not exist. Maybe not yet uploaded. Try again.
err = fmt.Errorf("Config not found")
time.Sleep(1 * time.Second)
continue
}
break
}
if err != nil {
log.Fatalf(
"Error trying to get config for the first time from consul: %v\n",
err)
}
updateConsulFlags(pair.Value)
go monitorConsulGoroutine(kv, service, qm.LastIndex)
}
// Write state to key with value.
func (d *ConsulStateDriver) Write(key string, value []byte) error {
key = processKey(key)
_, err := d.Client.KV().Put(&api.KVPair{Key: key, Value: value}, nil)
if err != nil && (api.IsServerError(err) || strings.Contains(err.Error(), "EOF") ||
strings.Contains(err.Error(), "connection refused")) {
for i := 0; i < maxConsulRetries; i++ {
_, err = d.Client.KV().Put(&api.KVPair{Key: key, Value: value}, nil)
if err == nil {
break
}
// Retry after a delay
time.Sleep(time.Second)
}
}
return err
}
// DelObj deletes an object
func (cp *ConsulClient) DelObj(key string) error {
key = processKey("/contiv.io/obj/" + processKey(key))
_, err := cp.client.KV().Delete(key, nil)
if err != nil {
if api.IsServerError(err) || strings.Contains(err.Error(), "EOF") ||
strings.Contains(err.Error(), "connection refused") {
for i := 0; i < maxConsulRetries; i++ {
_, err = cp.client.KV().Delete(key, nil)
if err == nil {
break
}
// Retry after a delay
time.Sleep(time.Second)
}
}
}
return err
}
// GetObj reads the object
func (cp *ConsulClient) GetObj(key string, retVal interface{}) error {
key = processKey("/contiv.io/obj/" + processKey(key))
resp, _, err := cp.client.KV().Get(key, &api.QueryOptions{RequireConsistent: true})
if err != nil {
if api.IsServerError(err) || strings.Contains(err.Error(), "EOF") ||
strings.Contains(err.Error(), "connection refused") {
for i := 0; i < maxConsulRetries; i++ {
resp, _, err = cp.client.KV().Get(key, &api.QueryOptions{RequireConsistent: true})
if err == nil {
break
}
// Retry after a delay
time.Sleep(time.Second)
}
}
// return error if it failed after retries
if err != nil {
return err
}
}
// Consul returns success and a nil kv when a key is not found,
// translate it to 'Key not found' error
if resp == nil {
return errors.New("Key not found")
}
// Parse JSON response
if err := json.Unmarshal(resp.Value, retVal); err != nil {
log.Errorf("Error parsing object %v, Err %v", resp.Value, err)
return err
}
return nil
}
func outputFile(kv *consulapi.KV, path, prefix string) {
vs, _, err := kv.List(prefix, nil)
if err != nil {
if consulapi.IsServerError(err) && outRetries < 10 {
log.Println("consul server err: retry after 1s")
<-time.After(time.Second)
outRetries++
outputFile(kv, path, prefix)
return
}
log.Fatal(err.Error())
}
f, err := os.Create(path)
if err != nil {
log.Fatal(err.Error())
}
defer f.Close()
for _, val := range vs {
fmt.Fprintf(f, "%s,%s\n", val.Key, val.Value)
}
f.Sync()
}
// ListDir returns a list of keys in a directory
func (cp *ConsulClient) ListDir(key string) ([]string, error) {
key = processKey("/contiv.io/obj/" + processKey(key))
kvs, _, err := cp.client.KV().List(key, nil)
if err != nil {
if api.IsServerError(err) || strings.Contains(err.Error(), "EOF") ||
strings.Contains(err.Error(), "connection refused") {
for i := 0; i < maxConsulRetries; i++ {
kvs, _, err = cp.client.KV().List(key, nil)
if err == nil {
break
}
// Retry after a delay
time.Sleep(time.Second)
}
}
// return error if it failed after retries
if err != nil {
return nil, err
}
}
// Consul returns success and a nil kv when a key is not found,
// translate it to 'Key not found' error
if kvs == nil {
return []string{}, nil
}
var keys []string
for _, kv := range kvs {
keys = append(keys, string(kv.Value))
}
return keys, nil
}
// WatchService watches for service instance changes
func (cp *ConsulClient) WatchService(srvName string, eventCh chan WatchServiceEvent, stopCh chan bool) error {
keyName := "contiv.io/service/" + srvName + "/"
// Run in background
go func() {
var currSrvMap = make(map[string]ServiceInfo)
// Get current list of services
srvList, lastIdx, err := cp.getServiceInstances(keyName, 0)
if err != nil {
log.Errorf("Error getting service instances for (%s): Err: %v", srvName, err)
} else {
// for each instance trigger an add event
for _, srvInfo := range srvList {
eventCh <- WatchServiceEvent{
EventType: WatchServiceEventAdd,
ServiceInfo: srvInfo,
}
// Add the service to local cache
srvKey := srvInfo.HostAddr + ":" + strconv.Itoa(srvInfo.Port)
currSrvMap[srvKey] = srvInfo
}
}
// Loop till asked to stop
for {
// Check if we should quit
select {
case <-stopCh:
return
default:
// Read the service instances
srvList, lastIdx, err = cp.getServiceInstances(keyName, lastIdx)
if err != nil {
if api.IsServerError(err) || strings.Contains(err.Error(), "EOF") || strings.Contains(err.Error(), "connection refused") {
log.Warnf("Consul service watch: server error: %v Retrying..", err)
} else {
log.Errorf("Error getting service instances for (%s): Err: %v. Exiting watch", srvName, err)
}
// Wait a little and continue
time.Sleep(5 * time.Second)
continue
} else {
log.Debugf("Got consul srv list: {%+v}. Curr: {%+v}", srvList, currSrvMap)
var newSrvMap = make(map[string]ServiceInfo)
// Check if there are any new services
for _, srvInfo := range srvList {
srvKey := srvInfo.HostAddr + ":" + strconv.Itoa(srvInfo.Port)
// If the entry didnt exists previously, trigger add event
if _, ok := currSrvMap[srvKey]; !ok {
log.Debugf("Sending add event for srv: %v", srvInfo)
eventCh <- WatchServiceEvent{
EventType: WatchServiceEventAdd,
ServiceInfo: srvInfo,
}
}
// create new service map
newSrvMap[srvKey] = srvInfo
}
// for all entries in old service list, see if we need to delete any
for _, srvInfo := range currSrvMap {
srvKey := srvInfo.HostAddr + ":" + strconv.Itoa(srvInfo.Port)
// if the entry does not exists in new list, delete it
if _, ok := newSrvMap[srvKey]; !ok {
log.Debugf("Sending delete event for srv: %v", srvInfo)
eventCh <- WatchServiceEvent{
EventType: WatchServiceEventDel,
ServiceInfo: srvInfo,
}
}
}
// set new srv map as the current
currSrvMap = newSrvMap
}
}
}
}()
return nil
}