Example #1
1
File: main.go Project: csigo/config
// Pusher push the localRepo to etcd.
func Pusher(etcdConn *etcd.Client, root, etcdRoot string) {
	fmt.Printf("Push config from local dir <%s> to etcd dir <%s>\n", root, etcdRoot)

	checkGitVersion()

	// check if root is symbolic link and convert it
	if r, err := filepath.EvalSymlinks(root); err == nil {
		fmt.Printf("convert symbolic link root %s to %s\n", root, r)
		root = r
	}

	// cd to repo or fatal
	cdRepo(root)

	// get local repo root
	repoRoot := runSingleCmdOrFatal("git rev-parse --show-toplevel")
	fmt.Printf("local repo root: %s\n", repoRoot)

	// get repo name
	repo := path.Base(repoRoot)
	fmt.Printf("using repo: %s\n", repo)

	// get path to git root
	pathToRepo, err := filepath.Rel(repoRoot, root)
	if err != nil {
		log.Fatalf("unable to find relative path from <%s> to <%s>, err %s", repoRoot, root, err)
	}

	// get branch name
	branch := runSingleCmdOrFatal("git rev-parse --abbrev-ref HEAD")
	// TODO: may need to relax this constrain.
	/*
		if branch != allowedPusherBranch {
			log.Fatalf("only %s branch is allowed to push, whereas you currently are in %s", allowedPusherBranch, branch)
		}
	*/
	fmt.Printf("using branch: %s\n", branch)

	// get commit hash
	commitHash := runSingleCmdOrFatal("git rev-parse HEAD")
	fmt.Printf("using commit: %s\n", commitHash)

	// get timestamp, committer email and subject of a commit
	info := runSingleCmdOrFatal("git show --quiet --pretty=%at:%ce:%s " + commitHash)
	fmt.Printf("commit timestamp:email:subject: %s\n", info)
	tokens := strings.Split(info, ":")
	if len(tokens) < 3 {
		log.Fatalf("commit info is not in the timestamp:email:subject format:%s\n", info)
	}

	tsstr, email, subject := tokens[0], tokens[1], strings.Join(tokens[2:], ":")
	// convert tsstr to timestamp
	timestamp, _ := strconv.ParseInt(tsstr, 10, 64)

	// set etcd
	remoteRoot := strings.Trim(etcdRoot, " \t")
	if remoteRoot == "" {
		log.Fatalf("etcdRoot is empty\n")
	}
	infoPath := fmt.Sprintf(infoPrefix, remoteRoot)

	// check config root on etcd
	resp, err := etcdConn.Get(remoteRoot, true, false)
	if err != nil {
		log.Fatalf("error testing remoteRoot, %s: %s\n", remoteRoot, err)
	}
	log.Infof("remoteRoot %s verified\n", remoteRoot)

	// default etcdLastCommit is current repo newest commit
	prevCFG := &config.ConfigInfo{}
	etcdLastCommit := []byte(runSingleCmdOrFatal("git rev-list --max-parents=0 HEAD"))
	etcdHasCommit := false
	log.Infof("reading last pushed information for %s", infoPath)
	resp, err = etcdConn.Get(infoPath, true, false)
	if err == nil {
		jerr := json.Unmarshal([]byte(resp.Node.Value), prevCFG)
		if jerr == nil && prevCFG.Version != "" {
			// read previous commit
			etcdLastCommit = []byte(prevCFG.Version)
			etcdHasCommit = true
		}
	} else {
		// previos config is empty or invalid
		log.Infof("previous configInfo doesn't exist")
	}

	// TODO: empty etcdLastCommit cause diff-tree to only print files of current commit.
	filestr := runSingleCmdOrFatal(fmt.Sprintf("git diff-tree --no-commit-id --name-status -r %s %s %s", etcdLastCommit, commitHash, root))
	// filter out files that are descendents of the rootpath
	modFiles := filterRelatedFiles(root, repoRoot, filestr)
	if len(*modFiles) == 0 {
		promptUser("Remote repo seems to be already up-to-date "+
			"(remote commit %s vs local commit %s). Do you want to continue?",
			etcdLastCommit, commitHash)
	} else {
		fmt.Println("This push will notify the following file changes:")
		for _, m := range *modFiles {
			fmt.Printf("%+v\n", m)
		}
		fmt.Println("End of notify changes.")
	}

	if etcdHasCommit {
		fmt.Printf("Remote commit: %s\n",
			runSingleCmdOrFatal("git log --format=%h:%s(%aN) -n 1 "+string(etcdLastCommit)))
	}
	fmt.Printf("Local commit:  %s\n",
		runSingleCmdOrFatal("git log --format=%h:%s(%aN) -n 1 "+commitHash))
	fmt.Printf("Total commit(s) to be pushed: %s\n",
		runSingleCmdOrFatal("git rev-list --count "+commitHash+" ^"+string(etcdLastCommit)))
	fmt.Printf("Files changed between remote/local commits:\n%s\n",
		runSingleCmdOrFatal("git diff --name-only "+string(etcdLastCommit)+" "+commitHash+" "+root))
	promptUser("Ready to push?")

	// walk through the repo and save content
	content := make(map[string]fileInfo)

	// since Walk will pass in absolute path, we are going to take
	// the root out from the beginning
	sz := len(root)

	walkFunc := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			log.Fatalf("Unable to traverse file %s: %s", path, err)
		}

		// TODO: we no need to scan all the files in the localRepo. Just check the files which are tracked by git: git ls-files
		// ignore .xxx file
		base := filepath.Base(path)
		// skip hidden dir, e.g. .git
		if string(base[0]) == "." {
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		bs := []byte{}
		// for dir we set "" content
		if !info.IsDir() {
			bs, err = ioutil.ReadFile(path)
			if err != nil {
				log.Fatalf("error reading file %s: %s\n", path, err)
			}
			bs = bytes.TrimRight(bs, "\n\r")
		}

		// skip root
		if path == root {
			return nil
		}
		// remove root path
		path = path[sz:]
		content[path] = fileInfo{isDir: info.IsDir(), content: bs}
		return nil
	}

	if err = filepath.Walk(root, walkFunc); err != nil {
		log.Fatalf("error when traversing %s: %s\n", root, err)
	}

	configInfo := config.ConfigInfo{
		Repo:    repo,
		Branch:  branch,
		Version: commitHash,
		Commit: config.CommitInfo{
			TimeStamp:      timestamp,
			CommitterEmail: email,
			Subject:        subject,
		},
		ModFiles:   *modFiles,
		PathToRepo: pathToRepo,
	}

	jsonBytes, err := json.Marshal(configInfo)
	if err != nil {
		log.Fatalf("error when marshal configInfo, err: %s", err)
	}
	fmt.Printf("ConfigInfo json: %s\n", string(jsonBytes[:len(jsonBytes)]))

	// Get previous pusher information from root node for sanity check.
	if !etcdHasCommit {
		promptUser("There is no existing commit in remote tree. It could because " +
			"you're pushing to a clean tree. Do you want to continue?")
	} else {
		// Sanity check of pusher info.
		if configInfo.Repo != prevCFG.Repo {
			promptUser("Repo to be pushed <%s> != remote repo name <%s>. Continue?",
				configInfo.Repo, prevCFG.Repo)
		}
		if configInfo.Branch != prevCFG.Branch {
			promptUser("Branch to be pushed <%s> != remote branch <%s>. Continue?",
				configInfo.Branch, prevCFG.Branch)
		}
		if configInfo.PathToRepo != prevCFG.PathToRepo {
			promptUser("Path to repo <%s> != remote path <%s>. Continue?",
				configInfo.PathToRepo, prevCFG.PathToRepo)
		}
	}

	keys := make([]string, len(content))
	// sort content's keys
	i := 0
	for k := range content {
		keys[i] = k
		i++
	}
	sort.Strings(keys)

	// set etcd content.
	for _, k := range keys {
		fileP := filepath.Join(remoteRoot, k)
		fmt.Printf("creating or setting %s\n", fileP)
		if content[k].isDir {
			if _, err := etcdConn.Get(fileP, true, false); err != nil {
				// dir doesn't exist
				resp, err = etcdConn.SetDir(fileP, 0)
			}
		} else {
			resp, err = etcdConn.Set(fileP, string(content[k].content), 0)
		}
		if err != nil {
			log.Fatalf("error when setting znode >%s(%s + %s)<. Config server will be inconsistent: %s",
				fileP, remoteRoot, k, err)
		}
	}

	// go over deletes in commit
	for _, mod := range *modFiles {
		if strings.ToLower(mod.Op) != "d" {
			continue
		}
		// it's a delete
		// Find the relative path to etcd root.
		fileP := filepath.Join(remoteRoot, mod.Path)
		fmt.Printf("deleting %s (%s + %s)\n", fileP, remoteRoot, mod.Path)
		_, err = etcdConn.Delete(fileP, true)
		if err != nil {
			log.Errorf("error deleting file >%s<. Will continue. error: %s\n", fileP, err)
		}
		// since git uses a file as  a commit unit, there is nothing to do with folder.
		// what we are going to do is to delete each fileP which is modified with "d" and its corresponding folder.
		// If we still have children under that folder, deleting the folder will fail but we do not care.
		pDir := filepath.Join(remoteRoot, path.Dir(mod.Path))
		_, err = etcdConn.DeleteDir(pDir)
		// In normal case, we should get an error.
		// so just logging, if we have no error here.
		if err == nil {
			log.Errorf("error deleting dir >%s<.\n", pDir)
		}
	}

	// touch the root node with commit info
	_, err = etcdConn.Set(infoPath, string(jsonBytes), 0)
	if err != nil {
		log.Fatalf("error setting remoteRoot >%s<: %s\n", remoteRoot, err)
	}
	fmt.Printf("All done\n")
}
Example #2
0
func etcdClientSetUp(t *testing.T, rawClient *etcd.Client) map[string]interface{} {
	var err error
	if _, err = rawClient.SetDir("test", 0); err != nil {
		t.Fatal(err)
	}
	if _, err = rawClient.Set("test/index", "1", 0); err != nil {
		t.Fatal(err)
	}
	if _, err = rawClient.SetDir("test/values", 0); err != nil {
		t.Fatal(err)
	}
	if _, err = rawClient.Set("test/values/a", "aye", 0); err != nil {
		t.Fatal(err)
	}
	if _, err = rawClient.Set("test/values/b", "bee", 0); err != nil {
		t.Fatal(err)
	}

	want := make(map[string]interface{})
	wantValues := make(map[string]interface{})
	wantValues["a"] = "aye"
	wantValues["b"] = "bee"
	want["values"] = wantValues
	want["index"] = "1"
	return want
}
Example #3
0
func EtcdMkDir(client *etcd.Client, path string, ttl uint64) error {
	_, err := client.SetDir(path, ttl)
	if err != nil {
		return err
	}
	return nil
}
Example #4
0
func setDirCommandFunc(cmd *cobra.Command, args []string, client *etcd.Client) (*etcd.Response, error) {
	if len(args) == 0 {
		return nil, errors.New("Key required")
	}
	key := args[0]
	ttl := ttlFlag

	return client.SetDir(key, uint64(ttl))
}
Example #5
0
func RegisterService(client *etcd.Client, service services.Service, ttl time.Duration) {
	keyPrefix := fmt.Sprintf("/services/%s/%s/", service.Service, service.Id)
	var attributes map[string]string = make(map[string]string)
	attributes["address"] = service.Address
	for key, value := range service.Labels {
		if key == "address" {
			fmt.Println("WARNING: overriding address(%s) with label: %s", service.Address, value)
		}
		attributes[key] = value
	}

	// create service/id dir with ttl
	client.SetDir(keyPrefix, uint64(ttl.Seconds()))
	for key, value := range attributes {
		_, err := client.Set(keyPrefix+key, value, 0)
		if err != nil {
			fmt.Println(err)
			// abort current registration
			return
		}
	}
}
Example #6
0
func TestNodeFs(t *testing.T) {
	g := Goblin(t)

	g.Describe("File", func() {
		var etcd *etcdm.Client
		var fs testEtcdFsMount

		g.Before(func() {
			etcd = etcdm.NewClient([]string{testEtcdEndpoint})
		})

		g.BeforeEach(func() {
			etcd.RawDelete("/test", true, true)
			etcd.SetDir("/test", 0)
			fs = NewTestEtcdFsMount()
		})

		g.AfterEach(func() {
			fs.Unmount()
		})

		g.Describe("Open", func() {
			g.It("Should be supported", func() {
				if _, e := etcd.Set("/test/foo", "bar", 0); e != nil {
					g.Fail(e)
				}

				file, err := os.Open(fs.Path() + "/test/foo")

				if err != nil {
					g.Fail(err)
				}

				file.Close()
			})
		})
		g.Describe("Create", func() {
			g.It("Should be supported", func() {
				file, err := os.Create(fs.Path() + "/test/bar")

				if err != nil {
					g.Fail(err)
				}
				file.Close()

				if _, er := etcd.Get("/test/bar", false, false); er != nil {
					g.Fail(er)
				}
			})
		})
		g.Describe("Delete", func() {
			g.It("Should be supported", func() {
				etcd.Set("/test/barfoo", "lala", 0)

				err := os.Remove(fs.Path() + "/test/barfoo")

				if err != nil {
					g.Fail(err)
				}

				if _, er := etcd.Get("/test/barfoo", false, false); er == nil {
					g.Fail("The key [/test/barfoo] should not exist")
				}
			})
		})
		g.Describe("Read", func() {
			g.It("Should be supported", func() {
				etcd.Set("/test/bar", "foo", 0)

				data, err := ioutil.ReadFile(fs.Path() + "/test/bar")

				if err != nil {
					g.Fail(err)
				}

				g.Assert(string(data)).Equal("foo")
			})
		})
		g.Describe("Write", func() {
			g.It("Should be supported", func() {
				if err := ioutil.WriteFile(fs.Path()+"/test/foobar", []byte("hello world"), 0666); err != nil {
					g.Fail(err)
				}

				res, err := etcd.Get("/test/foobar", false, false)

				if err != nil {
					g.Fail(err)
				}

				g.Assert(res.Node.Value).Equal("hello world")
			})
		})
	})
}