func dispatchCommand(c *cli.Context, client *etcd.Client, cmd *command.Command) {
targets := c.StringSlice("target")
if targets == nil || len(targets) == 0 {
log.Warningln("no target set! try to send command to all registered host.")
targets = fetchHosts(c, client)
}
if targets == nil {
log.Fatalln("no target to send command.")
} else {
log.Infoln("send command to: ", targets)
}
for _, target := range targets {
key := fmt.Sprintf("/beacon/commands/single/%s/%s/",
target, cmd.Id)
if c.GlobalString("prefix") != "" {
key = fmt.Sprintf("/%s%s", strings.Trim(c.GlobalString("prefix"), "/"), key)
}
if _, err := client.Set(key, cmd.Marshal(), 0); err != nil {
log.WithFields(log.Fields{
"error": err.Error(),
}).Fatalln("send command failed.")
}
}
}
func RemoveService(client *etcd.Client, s *Service) error {
basekey := appendPrefix(fmt.Sprintf("/beacon/registry/%s/haproxy/%s",
s.Cluster, s.Proto), s.Prefix)
var key string
if s.Proto == "tcp" {
key = fmt.Sprintf("%s/%s", basekey, s.Name)
} else {
key = fmt.Sprintf("%s/%s/backends/%s", basekey, s.Name, s.Backend)
}
log.WithFields(log.Fields{
"key": key,
}).Debugln("start to delete a service key.")
if _, err := client.Delete(key, true); err != nil {
return err
}
log.WithFields(log.Fields{
"name": s.Name,
"backend": s.Backend,
"proto": s.Proto,
}).Infoln("service unregistered.")
return nil
}
func acquire(c *etcd.Client, key, value string, ttl uint64) (uint64, error) {
resp, err := c.Create(key, value, ttl)
if err != nil {
return 0, err
}
return resp.EtcdIndex, nil
}
func NewEtcdClient(addr string, dialTimeout time.Duration) (*EtcdClient, error) {
var c *api.Client
/*
var err error
if cert != "" && key != "" {
c, err = etcd.NewTLSClient(machines, cert, key, caCert)
if err != nil {
return &Client{c}, err
}
} else {
c = etcd.NewClient(machines)
}
*/
// machine addresses
machines := []string{addr}
// create custom client
c = api.NewClient(machines)
if !c.SetCluster(machines) {
return nil, errors.New("cannot connect to etcd cluster: " + addr)
}
// configure dial timeout
c.SetDialTimeout(dialTimeout)
return &EtcdClient{addr: addr, client: c}, nil
}
//Acquire will attempt to acquire the lock, if blocking is set to true it will wait forever to do so.
//Setting blocking to false would be the equivalent of a fictional TryAcquire, an immediate return
//if locking fails.
func Acquire(c *etcd.Client, key, value string, ttl uint64, blocking bool) (*Lock, error) {
index := uint64(0)
var err error
tryAcquire := true
LOOP:
for {
if tryAcquire {
index, err = acquire(c, key, value, ttl)
if err == nil {
break LOOP
}
if !blocking {
return nil, err
}
tryAcquire = false
}
resp, err := c.Watch(key, 0, false, nil, nil)
if err != nil {
return nil, err
}
if resp.Action != "compareAndSwap" {
tryAcquire = true
}
}
return &Lock{
c: c,
key: key,
value: value,
ttl: ttl,
index: index,
held: true,
}, nil
}
// NewEtcdClient returns an *etcd.Client with a connection to named machines.
// It returns an error if a connection to the cluster cannot be made.
func NewEtcdClient(machines []string, cert, key string, caCert string) (*Client, error) {
var c *goetcd.Client
var err error
if cert != "" && key != "" {
c, err = goetcd.NewTLSClient(machines, cert, key, caCert)
if err != nil {
return &Client{c}, err
}
} else {
c = goetcd.NewClient(machines)
}
// Configure the DialTimeout, since 1 second is often too short
c.SetDialTimeout(time.Duration(3) * time.Second)
maxConnectAttempts := 10
for attempt := 1; attempt <= maxConnectAttempts; attempt++ {
success := c.SetCluster(machines)
if success {
break
return &Client{c}, nil
}
if attempt == maxConnectAttempts {
break
return &Client{c}, errors.New("cannot connect to etcd cluster: " + strings.Join(machines, ","))
}
log.Info(fmt.Sprintf("[Attempt: %d] Attempting access to etcd after 5 second sleep", attempt))
time.Sleep(5 * time.Second)
}
return &Client{c}, nil
}
// getAddress will return the host:port address of the service taking care of
// the task that we want to talk to.
// Currently we grab the information from etcd every time. Local cache could be used.
// If it failed, e.g. network failure, it should return error.
func GetAddress(client *etcd.Client, name string, id uint64) (string, error) {
resp, err := client.Get(TaskMasterPath(name, id), false, false)
if err != nil {
return "", err
}
return resp.Node.Value, nil
}
func MustCreate(c *etcd.Client, logger *log.Logger, key, value string, ttl uint64) *etcd.Response {
resp, err := c.Create(key, value, ttl)
if err != nil {
logger.Panicf("Create failed. Key: %s, err: %v", key, err)
}
return resp
}
func GetAndWatchEpoch(client *etcd.Client, appname string, epochC chan uint64, stop chan bool) (uint64, error) {
resp, err := client.Get(EpochPath(appname), false, false)
if err != nil {
log.Fatal("etcdutil: can not get epoch from etcd")
}
ep, err := strconv.ParseUint(resp.Node.Value, 10, 64)
if err != nil {
return 0, err
}
receiver := make(chan *etcd.Response, 1)
go client.Watch(EpochPath(appname), resp.EtcdIndex+1, false, receiver, stop)
go func() {
for resp := range receiver {
if resp.Action != "compareAndSwap" && resp.Action != "set" {
continue
}
epoch, err := strconv.ParseUint(resp.Node.Value, 10, 64)
if err != nil {
log.Fatal("etcdutil: can't parse epoch from etcd")
}
epochC <- epoch
}
}()
return ep, nil
}
// latestRunningVersion retrieves the highest version of the application published
// to etcd. If no app has been published, returns 0.
func latestRunningVersion(client *etcd.Client, appName string) int {
r := regexp.MustCompile(appNameRegex)
if client == nil {
// FIXME: client should only be nil during tests. This should be properly refactored.
if appName == "ceci-nest-pas-une-app" {
return 3
}
return 0
}
resp, err := client.Get(fmt.Sprintf("/deis/services/%s", appName), false, true)
if err != nil {
// no app has been published here (key not found) or there was an error
return 0
}
var versions []int
for _, node := range resp.Node.Nodes {
match := r.FindStringSubmatch(node.Key)
// account for keys that may not be an application container
if match == nil {
continue
}
version, err := strconv.Atoi(match[2])
if err != nil {
log.Println(err)
return 0
}
versions = append(versions, version)
}
return max(versions)
}
func cleanup(r *bytes.Buffer, e *etcd.Client, ep *testProcess, dp *testProcess) {
if r != nil {
log.Debug("Writing report")
rpath := testDir + "/report.txt"
if err := ioutil.WriteFile(rpath, r.Bytes(), 0644); err != nil {
log.WithFields(log.Fields{
"error": err,
"func": "ioutil.WriteFile",
"path": rpath,
}).Warning("Could not write report")
}
}
if dp != nil {
log.Debug("Exiting cdhcpd")
_ = dp.finish()
time.Sleep(time.Second)
}
if e != nil {
log.Debug("Clearing test data")
if _, err := e.Delete("/lochness", true); err != nil {
log.WithFields(log.Fields{
"error": err,
"func": "etcd.Delete",
}).Warning("Could not clear test-created data from etcd")
}
time.Sleep(time.Second)
}
if ep != nil {
log.Debug("Exiting etcd")
_ = ep.finish()
}
log.Info("Done")
}
func etcdClientSetUp(t *testing.T, rawClient *etcd.Client) map[string]interface{} {
var err error
if _, err = rawClient.SetDir("test", 0); err != nil {
t.Fatal(err)
}
if _, err = rawClient.Set("test/index", "1", 0); err != nil {
t.Fatal(err)
}
if _, err = rawClient.SetDir("test/values", 0); err != nil {
t.Fatal(err)
}
if _, err = rawClient.Set("test/values/a", "aye", 0); err != nil {
t.Fatal(err)
}
if _, err = rawClient.Set("test/values/b", "bee", 0); err != nil {
t.Fatal(err)
}
want := make(map[string]interface{})
wantValues := make(map[string]interface{})
wantValues["a"] = "aye"
wantValues["b"] = "bee"
want["values"] = wantValues
want["index"] = "1"
return want
}
func addDNS(record string, service *kapi.Service, etcdClient *etcd.Client) error {
// if PortalIP is not set, a DNS entry should not be created
if !kapi.IsServiceIPSet(service) {
log.Printf("Skipping dns record for headless service: %s\n", service.Name)
return nil
}
for i := range service.Spec.Ports {
svc := skymsg.Service{
Host: service.Spec.PortalIP,
Port: service.Spec.Ports[i].Port,
Priority: 10,
Weight: 10,
Ttl: 30,
}
b, err := json.Marshal(svc)
if err != nil {
return err
}
// Set with no TTL, and hope that kubernetes events are accurate.
log.Printf("Setting DNS record: %v -> %s:%d\n", record, service.Spec.PortalIP, service.Spec.Ports[i].Port)
_, err = etcdClient.Set(skymsg.Path(record), string(b), uint64(0))
if err != nil {
return err
}
}
return nil
}
// SetKey is used to log key value pairs to stdout/etcd so that dray can pass
// them down to subsequent images as needed. By default keys are logged to
// stdout as ----BEGIN PANAMAX DATA----\nkey=value\n----END PANAMAX DATA----
// tags. If LOG_TO env var is set to etcd, then keys are logged to etcd.
// The etcd api is set using ETCD_API env variable.
func SetKey(key string, value string) error {
logTo := os.Getenv("LOG_TO")
if logTo == "" {
logTo = "stdout"
}
logTo = strings.ToLower(logTo)
if logTo == "etcd" {
log.Info("Logging Keys to etcd...")
var ec *etcd.Client
eIP := os.Getenv("ETCD_API")
if eIP == "" {
eIP = "172.17.42.1:4001"
}
eIP = fmt.Sprintf("http://%s", eIP)
ms := []string{eIP}
ec = etcd.NewClient(ms)
_, e := ec.Set(key, value, 0)
if e != nil {
return e
}
} else {
fmt.Printf("\n----BEGIN PANAMAX DATA----\n%s=%s\n----END PANAMAX DATA----\n", key, value)
}
return nil
}
func NewEtcd(prefixKey string, config *EtcdConfig) (*Etcd, error) {
var (
client *etcd.Client
err error
)
if config.CertFile != "" && config.KeyFile != "" {
if client, err = etcd.NewTLSClient(config.Machines, config.CertFile, config.KeyFile, config.CaFile); err != nil {
Logger.Error("Failed to connect to Etcd. Error: %+v.", err)
return nil, err
}
} else {
client = etcd.NewClient(config.Machines)
}
// Set the default value if not provided.
if config.EtcdConsistency == "" {
config.EtcdConsistency = etcd.STRONG_CONSISTENCY
}
if err = client.SetConsistency(config.EtcdConsistency); err != nil {
Logger.Error("Failed to set Etcd consitency. Error: %+v.", err)
return nil, err
}
return &Etcd{client: client, prefixKey: prefixKey}, nil
}
func EtcdMkDir(client *etcd.Client, path string, ttl uint64) error {
_, err := client.SetDir(path, ttl)
if err != nil {
return err
}
return nil
}
func getRuntimeConfFromEtcd(client *etcd.Client, etcd_path string) (*RuntimeConfig, error) {
resp, err := client.Get(etcd_path, false, false)
if err != nil {
return nil, fmt.Errorf("failed to get RuntimeConfig:%v", err)
}
r := bytes.NewReader([]byte(resp.Node.Value))
return ReadRuntimeConfig(r)
}
// SetEtcd sets an array of values into a test etcd instance.
func SetEtcd(t *testing.T, keys []string, values []string, c *etcd.Client) {
for i, key := range keys {
_, err := c.Set(key, values[i], 0)
if err != nil {
t.Fatal(err)
}
}
}
func (b *backends) Watch(client *etcd.Client) {
receiver := make(chan *etcd.Response)
go client.Watch(b.path, uint64(b.watchIndex), true, receiver, nil)
for {
resp := <-receiver
b.Update(resp.Node, resp.Action)
}
}
func beat(client *etcd.Client, ip string, conf *config.Config) {
for {
if _, err := client.Set(conf.Etcd_dir+"/"+ip, conf.Agent_id, uint64(conf.Heartbeat_interval*2)); err != nil {
log.Fatal(err)
}
fmt.Printf("%v beat sent.\n", time.Now())
time.Sleep(time.Second * time.Duration(conf.Heartbeat_interval))
}
}
func AnnounceSite(id string, site Site, client *etcd.Client) {
key := fmt.Sprintf("/nginx/sites/%s", id)
value := FormatSite(site)
client.Set(key, value, 60)
log.Println("announcing", id, "as", value)
time.Sleep(30 * time.Second)
go AnnounceSite(id, site, client)
}
func addLockDirChild(client *etcd.Client, key string) (*etcd.Response, error) {
hostname, err := os.Hostname()
if err != nil {
return nil, errgo.Notef(err, "fail to get hostname")
}
client.SyncCluster()
return client.AddChild(key, hostname, 0)
}
func setDirCommandFunc(cmd *cobra.Command, args []string, client *etcd.Client) (*etcd.Response, error) {
if len(args) == 0 {
return nil, errors.New("Key required")
}
key := args[0]
ttl := ttlFlag
return client.SetDir(key, uint64(ttl))
}
func unlock(c *etcd.Client) {
for {
_, err := c.Set("lock", "unlock", 0)
if err == nil {
return
}
fmt.Println(err)
}
}
// On any change we just refresh everything
func handleChange(etcdClient *etcd.Client) {
resp, err := etcdClient.Get(etcdWatchKey, false, true)
if err != nil {
fmt.Println(err)
return
}
// Go and fetch all the services in REDIS
redisClient := clients.RedisClient()
if (redisClient == nil) {
fmt.Println("Couldn't connect to redis")
return
}
defer redisClient.Close()
keys, err := redis.Strings(redisClient.Do("KEYS", "frontend:*"))
if err != nil {
fmt.Println(err)
return
}
// Now we delete everything in redis and add everything back in from etcd
redisClient.Send("MULTI")
if len(keys) > 0 {
redisClient.Send("DEL", redis.Args{}.AddFlat(keys)...)
}
for _, node := range resp.Node.Nodes {
// This first level is a frontend
split := strings.Split(node.Key, "/")
domain := split[len(split) - 1]
frontendKey := "frontend:" + domain
// Belt and braces - delete this key just in case its not already (but it should be)
redisClient.Send("DEL", frontendKey)
redisClient.Send("RPUSH", frontendKey, domain)
for _, instNode := range node.Nodes {
instsplit := strings.Split(instNode.Key, "/")
inst := instsplit[len(instsplit) - 1]
host := "http://" + inst
redisClient.Send("RPUSH", frontendKey, host)
}
}
// Should be everything
_, err = redisClient.Do("EXEC")
if err != nil {
fmt.Println(err)
return
}
fmt.Println("Resynced hipache from etcd")
}
func get(e *etcd.Client, q string, t uint16) ([]dns.RR, error) {
path := questionToPath(q, t)
r, err := e.Get(path, false, false)
if err != nil {
return nil, err
}
h := dns.RR_Header{Name: q, Rrtype: t, Class: dns.ClassINET, Ttl: 60} // Ttl is overridden
rr := parseValue(t, r.Node.Value, h)
return []dns.RR{rr}, nil
}
func getCurrentVersion(client *etcd.Client, key string) *etcd.Response {
for {
if resp, err := client.Get(key, false, false); err == nil {
// failed to fetch first value
return resp
} else {
time.Sleep(time.Second)
}
}
}
func newAdapter(client *etcd.Client, workPool *workpool.WorkPool) *ETCDStoreAdapter {
client.SetConsistency(etcd.STRONG_CONSISTENCY)
return &ETCDStoreAdapter{
client: client,
workPool: workPool,
inflightWatches: map[chan bool]bool{},
inflightWatchLock: &sync.Mutex{},
}
}
func getEtcdValueOrDefault(c *etcd.Client, key string, defaultValue string) string {
resp, err := c.Get(key, false, false)
if err != nil {
if strings.Contains(fmt.Sprintf("%v", err), "Key not found") {
return defaultValue
}
assert(err, "url")
}
return resp.Node.Value
}
// CanConnect checks a given SSH key against the list of authorized users in etcd
// to check if a user with a given key is allowed to connect. It takes in an active
// etcd.Client struct pointer and the ssh key to test and returns a username
// and boolean representing if they are allowed to connect.
func CanConnect(e *etcd.Client, sshkey string) (user string, allowed bool) {
reply, err := e.Get("/flitter/builder/users/", true, true)
if err != nil {
log.Printf("etcd: %s", err)
return "", false
}
keybit := strings.Split(sshkey, " ")[1]
fp := GetFingerprint(keybit)
for _, userdir := range reply.Node.Nodes {
for _, fpnode := range userdir.Nodes {
thisFpSplit := strings.Split(fpnode.Key, "/")
thisFp := thisFpSplit[len(thisFpSplit)-1]
if fp == thisFp {
userpath := strings.Split(userdir.Key, "/")
user := userpath[len(userpath)-1]
return user, true
}
}
}
return
}