Ejemplo n.º 1
0
// acquireOrRenewLease either races to acquire a new master lease, or update the existing master's lease
// returns true if we have the lease, and an error if one occurs.
// TODO: use the master election utility once it is merged in.
func (c *Config) acquireOrRenewLease(etcdClient *etcd.Client) (bool, error) {
	result, err := etcdClient.Get(c.key, false, false)
	if err != nil {
		if tools.IsEtcdNotFound(err) {
			// there is no current master, try to become master, create will fail if the key already exists
			_, err := etcdClient.Create(c.key, c.whoami, c.ttl)
			if err != nil {
				return false, err
			}
			c.lastLease = time.Now()
			return true, nil
		}
		return false, err
	}
	if result.Node.Value == c.whoami {
		glog.Infof("key already exists, we are the master (%s)", result.Node.Value)
		// we extend our lease @ 1/2 of the existing TTL, this ensures the master doesn't flap around
		if result.Node.Expiration.Sub(time.Now()) < time.Duration(c.ttl/2)*time.Second {
			_, err := etcdClient.CompareAndSwap(c.key, c.whoami, c.ttl, c.whoami, result.Node.ModifiedIndex)
			if err != nil {
				return false, err
			}
		}
		c.lastLease = time.Now()
		return true, nil
	}
	glog.Infof("key already exists, the master is %s, sleeping.", result.Node.Value)
	return false, nil
}
Ejemplo n.º 2
0
func GetIndex(client *etcd.Client, basedir string, jobname string, ip string) (int, error) {
	jobdir := "/jobs" + "/" + jobname
	response, err := client.Get(jobdir, true, true)
	if err == nil {
		for i := 0; i < response.Node.Nodes.Len(); i++ {
			if response.Node.Nodes[i].Value == ip {
				return i, nil
			}
		}
	}
	response, err = client.AddChild(jobdir, ip, 0)
	if err != nil {
		fmt.Printf("use etcd to get index error: %v\n", err)
		return 0, err
	}
	mykey := response.Node.Key
	response, err = client.Get(jobdir, true, true)
	if err != nil {
		fmt.Printf("get etcd jobdir error: %v\n", err)
		return 0, err
	}
	for i := 0; i < response.Node.Nodes.Len(); i++ {
		if response.Node.Nodes[i].Key == mykey {
			return i, nil
		}
	}
	// this line would never reach.
	return 0, errors.New("etcd add child error!")
}
Ejemplo n.º 3
0
// getAddress will return the host:port address of the service taking care of
// the task that we want to talk to.
// Currently we grab the information from etcd every time. Local cache could be used.
// If it failed, e.g. network failure, it should return error.
func GetAddress(client *etcd.Client, name string, id uint64) (string, error) {
	resp, err := client.Get(TaskMasterPath(name, id), false, false)
	if err != nil {
		return "", err
	}
	return resp.Node.Value, nil
}
Ejemplo n.º 4
0
// latestRunningVersion retrieves the highest version of the application published
// to etcd. If no app has been published, returns 0.
func latestRunningVersion(client *etcd.Client, appName string) int {
	r := regexp.MustCompile(appNameRegex)
	if client == nil {
		// FIXME: client should only be nil during tests. This should be properly refactored.
		if appName == "ceci-nest-pas-une-app" {
			return 3
		}
		return 0
	}
	resp, err := client.Get(fmt.Sprintf("/deis/services/%s", appName), false, true)
	if err != nil {
		// no app has been published here (key not found) or there was an error
		return 0
	}
	var versions []int
	for _, node := range resp.Node.Nodes {
		match := r.FindStringSubmatch(node.Key)
		// account for keys that may not be an application container
		if match == nil {
			continue
		}
		version, err := strconv.Atoi(match[2])
		if err != nil {
			log.Println(err)
			return 0
		}
		versions = append(versions, version)
	}
	return max(versions)
}
Ejemplo n.º 5
0
func GetAndWatchEpoch(client *etcd.Client, appname string, epochC chan uint64, stop chan bool) (uint64, error) {
	resp, err := client.Get(EpochPath(appname), false, false)
	if err != nil {
		log.Fatal("etcdutil: can not get epoch from etcd")
	}
	ep, err := strconv.ParseUint(resp.Node.Value, 10, 64)
	if err != nil {
		return 0, err
	}
	receiver := make(chan *etcd.Response, 1)
	go client.Watch(EpochPath(appname), resp.EtcdIndex+1, false, receiver, stop)
	go func() {
		for resp := range receiver {
			if resp.Action != "compareAndSwap" && resp.Action != "set" {
				continue
			}
			epoch, err := strconv.ParseUint(resp.Node.Value, 10, 64)
			if err != nil {
				log.Fatal("etcdutil: can't parse epoch from etcd")
			}
			epochC <- epoch
		}
	}()

	return ep, nil
}
Ejemplo n.º 6
0
func getRuntimeConfFromEtcd(client *etcd.Client, etcd_path string) (*RuntimeConfig, error) {
	resp, err := client.Get(etcd_path, false, false)
	if err != nil {
		return nil, fmt.Errorf("failed to get RuntimeConfig:%v", err)
	}
	r := bytes.NewReader([]byte(resp.Node.Value))
	return ReadRuntimeConfig(r)
}
Ejemplo n.º 7
0
// On any change we just refresh everything
func handleChange(etcdClient *etcd.Client) {

	resp, err := etcdClient.Get(etcdWatchKey, false, true)
	if err != nil {
		fmt.Println(err)
		return
	}

	// Go and fetch all the services in REDIS
	redisClient := clients.RedisClient()
	if (redisClient == nil) {
		fmt.Println("Couldn't connect to redis")
		return
	}
	defer redisClient.Close()

	keys, err := redis.Strings(redisClient.Do("KEYS", "frontend:*"))
	if err != nil {
		fmt.Println(err)
		return
	}

	// Now we delete everything in redis and add everything back in from etcd
	redisClient.Send("MULTI")
	if len(keys) > 0 {
		redisClient.Send("DEL", redis.Args{}.AddFlat(keys)...)
	}

	for _, node := range resp.Node.Nodes {
		// This first level is a frontend
		split := strings.Split(node.Key, "/")
		domain := split[len(split) - 1]

		frontendKey := "frontend:" + domain

		// Belt and braces - delete this key just in case its not already (but it should be)
		redisClient.Send("DEL", frontendKey)
		redisClient.Send("RPUSH", frontendKey, domain)

		for _, instNode := range node.Nodes {
			instsplit := strings.Split(instNode.Key, "/")
			inst := instsplit[len(instsplit) - 1]
			host := "http://" + inst
			redisClient.Send("RPUSH", frontendKey, host)
		}
	}

	// Should be everything
	_, err = redisClient.Do("EXEC")
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Println("Resynced hipache from etcd")
}
Ejemplo n.º 8
0
func getEtcdRoute(client *etcd.Client) *Route {
	hostResp, err := client.Get("/deis/logs/host", false, false)
	assert(err, "url")
	portResp, err := client.Get("/deis/logs/port", false, false)
	assert(err, "url")
	protocol := getEtcdValueOrDefault(client, "/deis/logs/protocol", "udp")
	host := fmt.Sprintf("%s:%s", hostResp.Node.Value, portResp.Node.Value)
	log.Printf("routing all to %s://%s", protocol, host)
	return &Route{ID: "etcd", Target: Target{Type: "syslog", Addr: host, Protocol: protocol}}
}
Ejemplo n.º 9
0
func getEtcdValueOrDefault(c *etcd.Client, key string, defaultValue string) string {
	resp, err := c.Get(key, false, false)
	if err != nil {
		if strings.Contains(fmt.Sprintf("%v", err), "Key not found") {
			return defaultValue
		}
		assert(err, "url")
	}
	return resp.Node.Value
}
Ejemplo n.º 10
0
func getCurrentVersion(client *etcd.Client, key string) *etcd.Response {
	for {
		if resp, err := client.Get(key, false, false); err == nil {
			// failed to fetch first value
			return resp
		} else {
			time.Sleep(time.Second)
		}
	}
}
Ejemplo n.º 11
0
func get(e *etcd.Client, q string, t uint16) ([]dns.RR, error) {
	path := questionToPath(q, t)
	r, err := e.Get(path, false, false)
	if err != nil {
		return nil, err
	}
	h := dns.RR_Header{Name: q, Rrtype: t, Class: dns.ClassINET, Ttl: 60} // Ttl is overridden
	rr := parseValue(t, r.Node.Value, h)
	return []dns.RR{rr}, nil
}
Ejemplo n.º 12
0
func loadConfig(client *etcd.Client, config *server.Config) error {
	// Override what isn't set yet from the command line.
	n, err := client.Get("/"+msg.PathPrefix+"/config", false, false)
	if err != nil {
		log.Printf("skydns: falling back to default configuration, could not read from etcd: %s", err)
		return nil
	}
	if err := json.Unmarshal([]byte(n.Node.Value), config); err != nil {
		return fmt.Errorf("failed to unmarshal config: %s", err.Error())
	}
	return nil
}
Ejemplo n.º 13
0
// WaitFreeTask blocks until it gets a hint of free task
func WaitFreeTask(client *etcd.Client, name string, logger *log.Logger) (uint64, error) {
	slots, err := client.Get(FreeTaskDir(name), false, true)
	if err != nil {
		return 0, err
	}
	if total := len(slots.Node.Nodes); total > 0 {
		ri := rand.Intn(total)
		s := slots.Node.Nodes[ri]
		idStr := path.Base(s.Key)
		id, err := strconv.ParseUint(idStr, 0, 64)
		if err != nil {
			return 0, err
		}
		logger.Printf("got free task %v, randomly choose %d to try...", ListKeys(slots.Node.Nodes), ri)
		return id, nil
	}

	watchIndex := slots.EtcdIndex + 1
	respChan := make(chan *etcd.Response, 1)
	go func() {
		for {
			logger.Printf("start to wait failure at index %d", watchIndex)
			resp, err := client.Watch(FreeTaskDir(name), watchIndex, true, nil, nil)
			if err != nil {
				logger.Printf("WARN: WaitFailure watch failed: %v", err)
				return
			}
			if resp.Action == "set" {
				respChan <- resp
				return
			}
			watchIndex = resp.EtcdIndex + 1
		}
	}()
	var resp *etcd.Response
	var waitTime uint64 = 0
	for {
		select {
		case resp = <-respChan:
			idStr := path.Base(resp.Node.Key)
			id, err := strconv.ParseUint(idStr, 10, 64)
			if err != nil {
				return 0, err
			}
			return id, nil
		case <-time.After(10 * time.Second):
			waitTime++
			logger.Printf("Node already wait failure for %d0s", waitTime)
		}
	}

}
Ejemplo n.º 14
0
Archivo: etcd.go Proyecto: richm/origin
// TestEtcdClient verifies a client is functional.  It will attempt to
// connect to the etcd server and block until the server responds at least once, or return an
// error if the server never responded.
func TestEtcdClient(etcdClient *etcdclient.Client) error {
	for i := 0; ; i++ {
		_, err := etcdClient.Get("/", false, false)
		if err == nil || etcdutil.IsEtcdNotFound(err) {
			break
		}
		if i > 100 {
			return fmt.Errorf("could not reach etcd: %v", err)
		}
		time.Sleep(50 * time.Millisecond)
	}
	return nil
}
Ejemplo n.º 15
0
func loop(channel chan *store.Response, etcdClient *etcd.Client, nodename string) {
	for {
		responses, err := etcdClient.Get(nodename)
		if err != nil {
			//print an error
		} else {
			for _, response := range responses {
				channel <- response
			}
		}
		time.Sleep(time.Second * 2)

	}
}
Ejemplo n.º 16
0
// Demarshal takes an etcd client and a an anonymous interface to
// seed with values from etcd. This will throw an error if there is an exceptional
// error in the etcd client or you are invoking this incorrectly with maps.
// Any missing keys in etcd will be filled in with blank strings.
func Demarshal(etcd *etcd.Client, target interface{}) (err error) {
	val := reflect.ValueOf(target).Elem()

	for i := 0; i < val.NumField(); i++ {
		valueField := val.Field(i)
		typeField := val.Type().Field(i)
		tag := typeField.Tag

		if tag.Get("etcd") == "" {
			continue
		}

		switch valueField.Kind() {
		case reflect.Bool:
			if _, notok := etcd.Get(tag.Get("etcd"), false, false); notok == nil {
				valueField.SetBool(true)
			} else {
				valueField.SetBool(false)
			}

		case reflect.String:
			etcdval, err := etcd.Get(tag.Get("etcd"), false, false)
			if err != nil {
				valueField.SetString("")
			}

			valueField.Set(reflect.ValueOf(etcdval.Node.Value))

		case reflect.Map:
			keyKind := typeField.Type.Key().Kind()

			if keyKind != reflect.String {
				return errors.New("Map must be string[string]")
			}

			resp, err := etcd.Get(tag.Get("etcd"), true, true)
			if err != nil {
				return err
			}

			if !resp.Node.Dir {
				return errors.New("maps must be pointed at an etcd directory")
			}

			SetMapOffDir(resp.Node, &valueField, "")
		}
	}

	return
}
Ejemplo n.º 17
0
func LoadConfig(client *etcd.Client) (*Config, error) {
	n, err := client.Get("/skydns/config", false, false)
	config := &Config{ReadTimeout: 0, WriteTimeout: 0, Domain: "", DnsAddr: "", Nameservers: []string{""}, DNSSEC: ""}
	if err != nil {
		return config, nil
	}
	if err := json.Unmarshal([]byte(n.Node.Value), &config); err != nil {
		return nil, err
	}
	if err := setDefaults(config); err != nil {
		return nil, err
	}
	return config, nil
}
Ejemplo n.º 18
0
func checkStats(cli *etcd.Client, conf *config.FiddlerConf, pool spawner.SpawnPool) (err error) {
	// Pull aggregates of the information from config
	serverResp, err := cli.Get("fiddler/servers")
	if err != nil {
		return err
	}

	serverCount := len(serverResp)
	grow, shrink := false, false // We'll track both

	if serverCount < conf.Scale.Max {
		// We might want to Grow

		// Let's see if we fit any of the parameters
		for stat, val := range conf.Scale.Grow {
			if pass, err := check(cli, stat, val); err != nil {
				return err
			} else if pass {
				grow = true
			}
		}
	}

	if serverCount > conf.Scale.Min {
		// We might want to Shrink

		// Let's see if we fit any of the parameters
		for stat, val := range conf.Scale.Shrink {
			if pass, err := check(cli, stat, val); err != nil {
				return err
			} else if pass {
				shrink = true
			}
		}
	}

	if grow && shrink {
		log.Println("Fail as we want to grow and shrink.")
	} else if grow {
		log.Println("I want to grow")
		// TODO: We need to track the spawning of instances
		// TODO: We need to come up with a heuristic of when to go.  Every spike != growth
		// pool.Grow()
	} else if shrink {
		log.Println("I want to shrink")
		// pool.Shrink()
	}

	return
}
Ejemplo n.º 19
0
func (etcd *ETCDClusterRunner) deleteDir(client *etcdclient.Client, dir string) {
	responses, err := client.Get(dir)
	Ω(err).ShouldNot(HaveOccured())
	for _, response := range responses {
		if response.Key != "/_etcd" {
			if response.Dir == true {
				etcd.deleteDir(client, response.Key)
			} else {
				_, err := client.Delete(response.Key)
				Ω(err).ShouldNot(HaveOccured())
			}
		}
	}
}
Ejemplo n.º 20
0
// get is a wrapper for client.Get that uses SingleInflight to suppress multiple
// outstanding queries.
func get(client *etcd.Client, path string, recursive bool) (*etcd.Response, error) {
	resp, err, _ := etcdInflight.Do(path, func() (*etcd.Response, error) {
		r, e := client.Get(path, false, recursive)
		if e != nil {
			return nil, e
		}
		return r, e
	})
	if err != nil {
		return resp, err
	}
	// shared?
	return resp, err
}
Ejemplo n.º 21
0
func loadConfig(client *etcd.Client, config *Config) (*Config, error) {
	n, err := client.Get("/bind/config", false, false)
	if err != nil {
		if err := setDefaults(config); err != nil {
			return nil, err
		}
		return config, nil
	}
	if err := json.Unmarshal([]byte(n.Node.Value), &config); err != nil {
		return nil, err
	}
	if err := setDefaults(config); err != nil {
		return nil, err
	}
	return config, nil
}
Ejemplo n.º 22
0
// etcdGetInitialWatchState turns an etcd Get request into a watch equivalent
func etcdGetInitialWatchState(client *etcd.Client, key string, recursive bool, incoming chan<- *etcd.Response) (resourceVersion uint64, err error) {
	resp, err := client.Get(key, false, recursive)
	if err != nil {
		if !etcdutil.IsEtcdNotFound(err) {
			glog.Errorf("watch was unable to retrieve the current index for the provided key (%q): %v", key, err)
			return resourceVersion, err
		}
		if etcdError, ok := err.(*etcd.EtcdError); ok {
			resourceVersion = etcdError.Index
		}
		return resourceVersion, nil
	}
	resourceVersion = resp.EtcdIndex
	convertRecursiveResponse(resp.Node, resp, incoming)
	return
}
Ejemplo n.º 23
0
func getStats(cli *etcd.Client, metric string) (values []float64, err error) {
	serverResp, err := cli.Get(path.Join("fiddler/stats", metric))
	if err != nil {
		return nil, err
	}

	values = make([]float64, len(serverResp))
	for i, resp := range serverResp {
		if values[i], err = strconv.ParseFloat(resp.Value, 64); err != nil {
			return nil, err
		}
	}
	log.Printf("Metric `%s` values: %v\n", metric, values)

	return
}
Ejemplo n.º 24
0
func (b *backends) Sync(client *etcd.Client) error {
	resp, err := client.Get(b.path, false, true)

	if err != nil {
		return err
	}

	for _, n := range resp.Node.Nodes {
		b.Update(&n, resp.Action)
	}

	// Begin the watch after this sync from the next sync
	b.watchIndex = resp.EtcdIndex + 1

	return nil
}
Ejemplo n.º 25
0
func WatchMeta(c *etcd.Client, path string, stop chan bool, responseHandler func(*etcd.Response)) error {
	resp, err := c.Get(path, false, false)
	if err != nil {
		return err
	}
	// Get previous meta. We need to handle it.
	if resp.Node.Value != "" {
		go responseHandler(resp)
	}
	receiver := make(chan *etcd.Response, 1)
	go c.Watch(path, resp.EtcdIndex+1, false, receiver, stop)
	go func(receiver chan *etcd.Response) {
		for resp := range receiver {
			responseHandler(resp)
		}
	}(receiver)
	return nil
}
Ejemplo n.º 26
0
// getCommandFunc executes the "get" command.
func getCommandFunc(cmd *cobra.Command, args []string, client *etcd.Client) (*etcd.Response, error) {
	if len(args) == 0 {
		return nil, errors.New("Key required")
	}
	key := args[0]
	consistent := getConsistentFlag
	sorted := getSortFlag

	// Setup consistency on the client.
	if consistent {
		client.SetConsistency(etcd.STRONG_CONSISTENCY)
	} else {
		client.SetConsistency(etcd.WEAK_CONSISTENCY)
	}

	// Retrieve the value from the server.
	return client.Get(key, sorted, false)
}
Ejemplo n.º 27
0
func (locker *EtcdLocker) acquire(client *etcd.Client, key string, ttl uint64, wait bool) (Lock, error) {
	hasLock := false
	key = addPrefix(key)
	lock, err := addLockDirChild(client, key)
	if err != nil {
		return nil, errgo.Mask(err)
	}

	for !hasLock {
		res, err := client.Get(key, true, true)
		if err != nil {
			return nil, errgo.Mask(err)
		}

		if len(res.Node.Nodes) > 1 {
			sort.Sort(res.Node.Nodes)
			if res.Node.Nodes[0].CreatedIndex != lock.Node.CreatedIndex {
				if !wait {
					client.Delete(lock.Node.Key, false)
					return nil, &Error{res.Node.Nodes[0].Value}
				} else {
					err = locker.Wait(lock.Node.Key)
					if err != nil {
						return nil, errgo.Mask(err)
					}
				}
			} else {
				// if the first index is the current one, it's our turn to lock the key
				hasLock = true
			}
		} else {
			// If there are only 1 node, it's our, lock is acquired
			hasLock = true
		}
	}

	// If we get the lock, set the ttl and return it
	_, err = client.Update(lock.Node.Key, lock.Node.Value, ttl)
	if err != nil {
		return nil, errgo.Mask(err)
	}

	return &EtcdLock{client, lock.Node.Key, lock.Node.CreatedIndex}, nil
}
Ejemplo n.º 28
0
func GetBackends(client *etcd.Client, service, backendName string) ([]Backend, error) {

	resp, err := client.Get(service, false, true)
	if err != nil {
		log.Println("Error when reading etcd: ", err)
		return nil, err
	} else {
		backends := make([]Backend, len(resp.Node.Nodes))
		for index, element := range resp.Node.Nodes {

			key := (*element).Key // key format is: /service/IP:PORT
			service := strings.Split(key[strings.LastIndex(key, "/")+1:], ":")

			backends[index] = Backend{Name: fmt.Sprintf("back-%v", index), Ip: service[0], Port: service[1]}
		}
		return backends, nil
	}

}
Ejemplo n.º 29
0
func Ls(client *etcd.Client, path string) ([]string, error) {
	result := make([]string, 0)
	response, err := client.Get(path, true, false)
	if err != nil {
		return nil, err
	}

	//文件,列出自身路径
	if response.Node.Nodes == nil {
		return append(result, response.Node.Value), nil
	}

	//目录,列出目录内容
	for _, n := range response.Node.Nodes {
		result = append(result, n.Value)

	}
	return result, nil
}
Ejemplo n.º 30
-1
// CanConnect checks a given SSH key against the list of authorized users in etcd
// to check if a user with a given key is allowed to connect. It takes in an active
// etcd.Client struct pointer and the ssh key to test and returns a username
// and boolean representing if they are allowed to connect.
func CanConnect(e *etcd.Client, sshkey string) (user string, allowed bool) {
	reply, err := e.Get("/flitter/builder/users/", true, true)

	if err != nil {
		log.Printf("etcd: %s", err)
		return "", false
	}

	keybit := strings.Split(sshkey, " ")[1]
	fp := GetFingerprint(keybit)

	for _, userdir := range reply.Node.Nodes {
		for _, fpnode := range userdir.Nodes {
			thisFpSplit := strings.Split(fpnode.Key, "/")
			thisFp := thisFpSplit[len(thisFpSplit)-1]

			if fp == thisFp {
				userpath := strings.Split(userdir.Key, "/")
				user := userpath[len(userpath)-1]
				return user, true
			}
		}
	}

	return
}