// Pusher push the localRepo to etcd. func Pusher(etcdConn *etcd.Client, root, etcdRoot string) { fmt.Printf("Push config from local dir <%s> to etcd dir <%s>\n", root, etcdRoot) checkGitVersion() // check if root is symbolic link and convert it if r, err := filepath.EvalSymlinks(root); err == nil { fmt.Printf("convert symbolic link root %s to %s\n", root, r) root = r } // cd to repo or fatal cdRepo(root) // get local repo root repoRoot := runSingleCmdOrFatal("git rev-parse --show-toplevel") fmt.Printf("local repo root: %s\n", repoRoot) // get repo name repo := path.Base(repoRoot) fmt.Printf("using repo: %s\n", repo) // get path to git root pathToRepo, err := filepath.Rel(repoRoot, root) if err != nil { log.Fatalf("unable to find relative path from <%s> to <%s>, err %s", repoRoot, root, err) } // get branch name branch := runSingleCmdOrFatal("git rev-parse --abbrev-ref HEAD") // TODO: may need to relax this constrain. /* if branch != allowedPusherBranch { log.Fatalf("only %s branch is allowed to push, whereas you currently are in %s", allowedPusherBranch, branch) } */ fmt.Printf("using branch: %s\n", branch) // get commit hash commitHash := runSingleCmdOrFatal("git rev-parse HEAD") fmt.Printf("using commit: %s\n", commitHash) // get timestamp, committer email and subject of a commit info := runSingleCmdOrFatal("git show --quiet --pretty=%at:%ce:%s " + commitHash) fmt.Printf("commit timestamp:email:subject: %s\n", info) tokens := strings.Split(info, ":") if len(tokens) < 3 { log.Fatalf("commit info is not in the timestamp:email:subject format:%s\n", info) } tsstr, email, subject := tokens[0], tokens[1], strings.Join(tokens[2:], ":") // convert tsstr to timestamp timestamp, _ := strconv.ParseInt(tsstr, 10, 64) // set etcd remoteRoot := strings.Trim(etcdRoot, " \t") if remoteRoot == "" { log.Fatalf("etcdRoot is empty\n") } infoPath := fmt.Sprintf(infoPrefix, remoteRoot) // check config root on etcd resp, err := etcdConn.Get(remoteRoot, true, false) if err != nil { log.Fatalf("error testing remoteRoot, %s: %s\n", remoteRoot, err) } log.Infof("remoteRoot %s verified\n", remoteRoot) // default etcdLastCommit is current repo newest commit prevCFG := &config.ConfigInfo{} etcdLastCommit := []byte(runSingleCmdOrFatal("git rev-list --max-parents=0 HEAD")) etcdHasCommit := false log.Infof("reading last pushed information for %s", infoPath) resp, err = etcdConn.Get(infoPath, true, false) if err == nil { jerr := json.Unmarshal([]byte(resp.Node.Value), prevCFG) if jerr == nil && prevCFG.Version != "" { // read previous commit etcdLastCommit = []byte(prevCFG.Version) etcdHasCommit = true } } else { // previos config is empty or invalid log.Infof("previous configInfo doesn't exist") } // TODO: empty etcdLastCommit cause diff-tree to only print files of current commit. filestr := runSingleCmdOrFatal(fmt.Sprintf("git diff-tree --no-commit-id --name-status -r %s %s %s", etcdLastCommit, commitHash, root)) // filter out files that are descendents of the rootpath modFiles := filterRelatedFiles(root, repoRoot, filestr) if len(*modFiles) == 0 { promptUser("Remote repo seems to be already up-to-date "+ "(remote commit %s vs local commit %s). Do you want to continue?", etcdLastCommit, commitHash) } else { fmt.Println("This push will notify the following file changes:") for _, m := range *modFiles { fmt.Printf("%+v\n", m) } fmt.Println("End of notify changes.") } if etcdHasCommit { fmt.Printf("Remote commit: %s\n", runSingleCmdOrFatal("git log --format=%h:%s(%aN) -n 1 "+string(etcdLastCommit))) } fmt.Printf("Local commit: %s\n", runSingleCmdOrFatal("git log --format=%h:%s(%aN) -n 1 "+commitHash)) fmt.Printf("Total commit(s) to be pushed: %s\n", runSingleCmdOrFatal("git rev-list --count "+commitHash+" ^"+string(etcdLastCommit))) fmt.Printf("Files changed between remote/local commits:\n%s\n", runSingleCmdOrFatal("git diff --name-only "+string(etcdLastCommit)+" "+commitHash+" "+root)) promptUser("Ready to push?") // walk through the repo and save content content := make(map[string]fileInfo) // since Walk will pass in absolute path, we are going to take // the root out from the beginning sz := len(root) walkFunc := func(path string, info os.FileInfo, err error) error { if err != nil { log.Fatalf("Unable to traverse file %s: %s", path, err) } // TODO: we no need to scan all the files in the localRepo. Just check the files which are tracked by git: git ls-files // ignore .xxx file base := filepath.Base(path) // skip hidden dir, e.g. .git if string(base[0]) == "." { if info.IsDir() { return filepath.SkipDir } return nil } bs := []byte{} // for dir we set "" content if !info.IsDir() { bs, err = ioutil.ReadFile(path) if err != nil { log.Fatalf("error reading file %s: %s\n", path, err) } bs = bytes.TrimRight(bs, "\n\r") } // skip root if path == root { return nil } // remove root path path = path[sz:] content[path] = fileInfo{isDir: info.IsDir(), content: bs} return nil } if err = filepath.Walk(root, walkFunc); err != nil { log.Fatalf("error when traversing %s: %s\n", root, err) } configInfo := config.ConfigInfo{ Repo: repo, Branch: branch, Version: commitHash, Commit: config.CommitInfo{ TimeStamp: timestamp, CommitterEmail: email, Subject: subject, }, ModFiles: *modFiles, PathToRepo: pathToRepo, } jsonBytes, err := json.Marshal(configInfo) if err != nil { log.Fatalf("error when marshal configInfo, err: %s", err) } fmt.Printf("ConfigInfo json: %s\n", string(jsonBytes[:len(jsonBytes)])) // Get previous pusher information from root node for sanity check. if !etcdHasCommit { promptUser("There is no existing commit in remote tree. It could because " + "you're pushing to a clean tree. Do you want to continue?") } else { // Sanity check of pusher info. if configInfo.Repo != prevCFG.Repo { promptUser("Repo to be pushed <%s> != remote repo name <%s>. Continue?", configInfo.Repo, prevCFG.Repo) } if configInfo.Branch != prevCFG.Branch { promptUser("Branch to be pushed <%s> != remote branch <%s>. Continue?", configInfo.Branch, prevCFG.Branch) } if configInfo.PathToRepo != prevCFG.PathToRepo { promptUser("Path to repo <%s> != remote path <%s>. Continue?", configInfo.PathToRepo, prevCFG.PathToRepo) } } keys := make([]string, len(content)) // sort content's keys i := 0 for k := range content { keys[i] = k i++ } sort.Strings(keys) // set etcd content. for _, k := range keys { fileP := filepath.Join(remoteRoot, k) fmt.Printf("creating or setting %s\n", fileP) if content[k].isDir { if _, err := etcdConn.Get(fileP, true, false); err != nil { // dir doesn't exist resp, err = etcdConn.SetDir(fileP, 0) } } else { resp, err = etcdConn.Set(fileP, string(content[k].content), 0) } if err != nil { log.Fatalf("error when setting znode >%s(%s + %s)<. Config server will be inconsistent: %s", fileP, remoteRoot, k, err) } } // go over deletes in commit for _, mod := range *modFiles { if strings.ToLower(mod.Op) != "d" { continue } // it's a delete // Find the relative path to etcd root. fileP := filepath.Join(remoteRoot, mod.Path) fmt.Printf("deleting %s (%s + %s)\n", fileP, remoteRoot, mod.Path) _, err = etcdConn.Delete(fileP, true) if err != nil { log.Errorf("error deleting file >%s<. Will continue. error: %s\n", fileP, err) } // since git uses a file as a commit unit, there is nothing to do with folder. // what we are going to do is to delete each fileP which is modified with "d" and its corresponding folder. // If we still have children under that folder, deleting the folder will fail but we do not care. pDir := filepath.Join(remoteRoot, path.Dir(mod.Path)) _, err = etcdConn.DeleteDir(pDir) // In normal case, we should get an error. // so just logging, if we have no error here. if err == nil { log.Errorf("error deleting dir >%s<.\n", pDir) } } // touch the root node with commit info _, err = etcdConn.Set(infoPath, string(jsonBytes), 0) if err != nil { log.Fatalf("error setting remoteRoot >%s<: %s\n", remoteRoot, err) } fmt.Printf("All done\n") }
func addDNS(record string, service *kapi.Service, etcdClient *etcd.Client) error { // if PortalIP is not set, a DNS entry should not be created if !kapi.IsServiceIPSet(service) { log.Printf("Skipping dns record for headless service: %s\n", service.Name) return nil } for i := range service.Spec.Ports { svc := skymsg.Service{ Host: service.Spec.PortalIP, Port: service.Spec.Ports[i].Port, Priority: 10, Weight: 10, Ttl: 30, } b, err := json.Marshal(svc) if err != nil { return err } // Set with no TTL, and hope that kubernetes events are accurate. log.Printf("Setting DNS record: %v -> %s:%d\n", record, service.Spec.PortalIP, service.Spec.Ports[i].Port) _, err = etcdClient.Set(skymsg.Path(record), string(b), uint64(0)) if err != nil { return err } } return nil }
func dispatchCommand(c *cli.Context, client *etcd.Client, cmd *command.Command) { targets := c.StringSlice("target") if targets == nil || len(targets) == 0 { log.Warningln("no target set! try to send command to all registered host.") targets = fetchHosts(c, client) } if targets == nil { log.Fatalln("no target to send command.") } else { log.Infoln("send command to: ", targets) } for _, target := range targets { key := fmt.Sprintf("/beacon/commands/single/%s/%s/", target, cmd.Id) if c.GlobalString("prefix") != "" { key = fmt.Sprintf("/%s%s", strings.Trim(c.GlobalString("prefix"), "/"), key) } if _, err := client.Set(key, cmd.Marshal(), 0); err != nil { log.WithFields(log.Fields{ "error": err.Error(), }).Fatalln("send command failed.") } } }
// SetKey is used to log key value pairs to stdout/etcd so that dray can pass // them down to subsequent images as needed. By default keys are logged to // stdout as ----BEGIN PANAMAX DATA----\nkey=value\n----END PANAMAX DATA---- // tags. If LOG_TO env var is set to etcd, then keys are logged to etcd. // The etcd api is set using ETCD_API env variable. func SetKey(key string, value string) error { logTo := os.Getenv("LOG_TO") if logTo == "" { logTo = "stdout" } logTo = strings.ToLower(logTo) if logTo == "etcd" { log.Info("Logging Keys to etcd...") var ec *etcd.Client eIP := os.Getenv("ETCD_API") if eIP == "" { eIP = "172.17.42.1:4001" } eIP = fmt.Sprintf("http://%s", eIP) ms := []string{eIP} ec = etcd.NewClient(ms) _, e := ec.Set(key, value, 0) if e != nil { return e } } else { fmt.Printf("\n----BEGIN PANAMAX DATA----\n%s=%s\n----END PANAMAX DATA----\n", key, value) } return nil }
// SetEtcd sets an array of values into a test etcd instance. func SetEtcd(t *testing.T, keys []string, values []string, c *etcd.Client) { for i, key := range keys { _, err := c.Set(key, values[i], 0) if err != nil { t.Fatal(err) } } }
func AnnounceSite(id string, site Site, client *etcd.Client) { key := fmt.Sprintf("/nginx/sites/%s", id) value := FormatSite(site) client.Set(key, value, 60) log.Println("announcing", id, "as", value) time.Sleep(30 * time.Second) go AnnounceSite(id, site, client) }
func beat(client *etcd.Client, ip string, conf *config.Config) { for { if _, err := client.Set(conf.Etcd_dir+"/"+ip, conf.Agent_id, uint64(conf.Heartbeat_interval*2)); err != nil { log.Fatal(err) } fmt.Printf("%v beat sent.\n", time.Now()) time.Sleep(time.Second * time.Duration(conf.Heartbeat_interval)) } }
func unlock(c *etcd.Client) { for { _, err := c.Set("lock", "unlock", 0) if err == nil { return } fmt.Println(err) } }
func setAll(c *etcd.Client, dir string, data map[string]string) error { for name, item := range data { _, err := c.Set(dir+name, item, 0) if err != nil { return err } } return nil }
// TrackStats is going to pull all pertinent system stats and update our state in etcd func TrackMyStats(cli *etcd.Client, myid string, stats []string) { for { cli.Set(path.Join("fiddler/servers", myid), "present", 60) for _, stat := range stats { // Get the value for the stat and set cli key for the value stat := NewStat(stat) stat.write(myid, cli) } time.Sleep(3 * time.Second) } }
// heartbeat to etcd cluster until stop func Heartbeat(client *etcd.Client, name string, taskID uint64, interval time.Duration, stop chan struct{}) error { for { _, err := client.Set(TaskHealthyPath(name, taskID), "health", computeTTL(interval)) if err != nil { return err } select { case <-time.After(interval): case <-stop: return nil } } }
// Create Etcd directory structure from a map, slice or struct. func Create(client *etcd.Client, path string, val reflect.Value) error { // fmt.Printf("# %s : %s : %s\n", path, val.Kind(), val.Type()) switch val.Kind() { case reflect.Ptr: orig := val.Elem() if !orig.IsValid() { return nil } if err := Create(client, path, orig); err != nil { return err } case reflect.Interface: orig := val.Elem() if err := Create(client, path, orig); err != nil { return err } case reflect.Struct: for i := 0; i < val.NumField(); i++ { t := val.Type().Field(i) k := t.Tag.Get("etcd") if err := Create(client, path+"/"+k, val.Field(i)); err != nil { return err } } case reflect.Map: for _, k := range val.MapKeys() { v := val.MapIndex(k) if err := Create(client, path+"/"+k.String(), v); err != nil { return err } } case reflect.Slice: for i := 0; i < val.Len(); i++ { Create(client, fmt.Sprintf("%s/%d", path, i), val.Index(i)) } case reflect.String: if _, err := client.Set(path, val.String(), 0); err != nil { return err } case reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Float32, reflect.Float64: if _, err := client.Set(path, fmt.Sprintf("%v", val.Interface()), 0); err != nil { return err } default: return fmt.Errorf("unsupported type: %s for path: %s", val.Kind(), path) } return nil }
func etcd_SetData(key string, value string) { info := config.GetServerInfo() log.Printf("info : %s\n", info.AccessString) var client *etcd.Client = nil client = etcd.NewClient(info.AccessString) _, err := client.Set(key, value, 0) if err != nil { log.Fatal(err) log.Printf("etcd_Setdata ERROR") } log.Printf("Current Set information: %s\n", value) }
func TryOccupyTask(client *etcd.Client, name string, taskID uint64, connection string) (bool, error) { _, err := client.Create(TaskHealthyPath(name, taskID), "health", 3) if err != nil { if strings.Contains(err.Error(), "Key already exists") { return false, nil } return false, err } idStr := strconv.FormatUint(taskID, 10) client.Delete(FreeTaskPath(name, idStr), false) _, err = client.Set(TaskMasterPath(name, taskID), connection, 0) if err != nil { return false, err } return true, nil }
func addDNS(record string, service *kapi.Service, etcdClient *etcd.Client) error { svc := skymsg.Service{ Host: service.Spec.PortalIP, Port: service.Spec.Port, Priority: 10, Weight: 10, Ttl: 30, } b, err := json.Marshal(svc) if err != nil { return err } // Set with no TTL, and hope that kubernetes events are accurate. log.Printf("Setting dns record: %v -> %s:%d\n", record, service.Spec.PortalIP, service.Spec.Port) _, err = etcdClient.Set(skymsg.Path(record), string(b), uint64(0)) return err }
// setCommandFunc executes the "set" command. func setCommandFunc(cmd *cobra.Command, args []string, client *etcd.Client) (*etcd.Response, error) { if len(args) == 0 { return nil, errors.New("Key required") } key := args[0] value, err := argOrStdin(args, os.Stdin, 1) if err != nil { return nil, errors.New("Value required") } ttl := ttlFlag prevValue := swapWithValueFlag prevIndex := swapWithIndexFlag if prevValue == "" && prevIndex == 0 { return client.Set(key, value, uint64(ttl)) } else { return client.CompareAndSwap(key, value, uint64(ttl), prevValue, uint64(prevIndex)) } }
func etcdClientSetUp(t *testing.T, rawClient *etcd.Client) map[string]interface{} { var err error if _, err = rawClient.SetDir("test", 0); err != nil { t.Fatal(err) } if _, err = rawClient.Set("test/index", "1", 0); err != nil { t.Fatal(err) } if _, err = rawClient.SetDir("test/values", 0); err != nil { t.Fatal(err) } if _, err = rawClient.Set("test/values/a", "aye", 0); err != nil { t.Fatal(err) } if _, err = rawClient.Set("test/values/b", "bee", 0); err != nil { t.Fatal(err) } want := make(map[string]interface{}) wantValues := make(map[string]interface{}) wantValues["a"] = "aye" wantValues["b"] = "bee" want["values"] = wantValues want["index"] = "1" return want }
func AddService(client *etcd.Client, s *Service) error { basekey := appendPrefix(fmt.Sprintf("/beacon/registry/%s/haproxy/%s/%s", s.Cluster, s.Proto, s.Name), s.Prefix) k := basekey + "/listen" log.WithFields(log.Fields{ "key": k, "value": s.Listen, }).Debugln("start to add a new service key.") if _, err := client.Set(k, s.Listen, 0); err != nil { return err } if s.Proto == "http" { if s.Backend == "" { return errors.New("backend name is required.") } hostDir := fmt.Sprintf("%s/backends/%s/hosts", basekey, s.Backend) if s.Hosts != nil { for _, host := range s.Hosts { log.WithFields(log.Fields{ "key": hostDir, "host": host, }).Debugln("register new host.") if _, err := client.Set(fmt.Sprintf("%s/%s", hostDir, host), "", 0); err != nil { return err } } } } log.WithFields(log.Fields{ "name": s.Name, "backend": s.Backend, "proto": s.Proto, "listen": s.Listen, "hosts": s.Hosts, }).Infoln("service registered.") return nil }
func RegisterService(client *etcd.Client, service services.Service, ttl time.Duration) { keyPrefix := fmt.Sprintf("/services/%s/%s/", service.Service, service.Id) var attributes map[string]string = make(map[string]string) attributes["address"] = service.Address for key, value := range service.Labels { if key == "address" { fmt.Println("WARNING: overriding address(%s) with label: %s", service.Address, value) } attributes[key] = value } // create service/id dir with ttl client.SetDir(keyPrefix, uint64(ttl.Seconds())) for key, value := range attributes { _, err := client.Set(keyPrefix+key, value, 0) if err != nil { fmt.Println(err) // abort current registration return } } }
// IsEtcdHealthy indicates whether etcd can respond to requests. func IsEtcdHealthy(client *etcd.Client) (ok bool, err error) { fakeID, err := id.Generate() if err != nil { return false, fmt.Errorf("Error generating health check key: %s", err) } key, expected := "status_"+fakeID, "test" if _, err = client.Set(key, expected, uint64(3*time.Second)); err != nil { return false, fmt.Errorf("Error storing health check key %#v: %s", key, err) } resp, err := client.Get(key, false, false) if err != nil { return false, fmt.Errorf("Error fetching health check key %#v: %s", key, err) } if resp.Node.Value != expected { return false, fmt.Errorf( "Unexpected value for health check key %#v: got %s; want %s", key, resp.Node.Value, expected) } return true, nil }
func NewValueSync(action string, config ValueConfig, client *etcd.Client, task *Task) *ValueSync { var ( res *etcd.Response err error ) switch action { case GET: res, err = client.Get(config.Path, true, true) if err != nil { log.Printf("Error:: %s [Get] - %s", config.Name, err) } case SET: _, err := client.Get(config.Path, true, true) if (err != nil) || config.ForceSet { log.Printf("Setting: %s - %s", config.Path, config.Value) res, err = client.Set(config.Path, config.Value, config.TTL) if err != nil { log.Printf("Error:: %s [Set] - %s", config.Name, err) } } } if res == nil { res = &etcd.Response{} } sync := &ValueSync{ Task: task, Config: config, Client: client, Action: action, Path: config.Path, } sync.UpdateTaskVars(res) return sync }
func setEtcd(client *etcd.Client, key, value string, ttl uint64) { _, err := client.Set(key, value, ttl) if err != nil && !strings.Contains(err.Error(), "Key already exists") { log.Println(err) } }
func (stat *Stat) write(myid string, cli *etcd.Client) { stat.Value = stat.GetStatValue() value := strconv.FormatFloat(stat.Value, 'f', -1, 64) cli.Set(path.Join("fiddler/stats", stat.StatType, myid), value, 60) }
func SetJobStatus(client *etcd.Client, name string, status int) error { _, err := client.Set(JobStatusPath(name), "done", 0) return err }
func announce(key string, server Server, etc *etcd.Client) { marshal, _ := json.Marshal(server) value := string(marshal) etc.Set(key, value, 10) }
func setDefaultEtcd(client *etcd.Client, key, value string) { _, err := client.Set(key, value, 0) if err != nil { log.Warn(err) } }
func setEtcd(client *etcd.Client, key, value string, ttl uint64) { _, err := client.Set(key, value, ttl) if err != nil { log.Warn(err) } }
// report failure to etcd cluster // If a framework detects a failure, it tries to report failure to /FreeTasks/{taskID} func ReportFailure(client *etcd.Client, name, failedTask string) error { _, err := client.Set(FreeTaskPath(name, failedTask), "failed", 0) return err }
func TestNodeFs(t *testing.T) { g := Goblin(t) g.Describe("File", func() { var etcd *etcdm.Client var fs testEtcdFsMount g.Before(func() { etcd = etcdm.NewClient([]string{testEtcdEndpoint}) }) g.BeforeEach(func() { etcd.RawDelete("/test", true, true) etcd.SetDir("/test", 0) fs = NewTestEtcdFsMount() }) g.AfterEach(func() { fs.Unmount() }) g.Describe("Open", func() { g.It("Should be supported", func() { if _, e := etcd.Set("/test/foo", "bar", 0); e != nil { g.Fail(e) } file, err := os.Open(fs.Path() + "/test/foo") if err != nil { g.Fail(err) } file.Close() }) }) g.Describe("Create", func() { g.It("Should be supported", func() { file, err := os.Create(fs.Path() + "/test/bar") if err != nil { g.Fail(err) } file.Close() if _, er := etcd.Get("/test/bar", false, false); er != nil { g.Fail(er) } }) }) g.Describe("Delete", func() { g.It("Should be supported", func() { etcd.Set("/test/barfoo", "lala", 0) err := os.Remove(fs.Path() + "/test/barfoo") if err != nil { g.Fail(err) } if _, er := etcd.Get("/test/barfoo", false, false); er == nil { g.Fail("The key [/test/barfoo] should not exist") } }) }) g.Describe("Read", func() { g.It("Should be supported", func() { etcd.Set("/test/bar", "foo", 0) data, err := ioutil.ReadFile(fs.Path() + "/test/bar") if err != nil { g.Fail(err) } g.Assert(string(data)).Equal("foo") }) }) g.Describe("Write", func() { g.It("Should be supported", func() { if err := ioutil.WriteFile(fs.Path()+"/test/foobar", []byte("hello world"), 0666); err != nil { g.Fail(err) } res, err := etcd.Get("/test/foobar", false, false) if err != nil { g.Fail(err) } g.Assert(res.Node.Value).Equal("hello world") }) }) }) }