// Pusher push the localRepo to etcd.
func Pusher(etcdConn *etcd.Client, root, etcdRoot string) {
fmt.Printf("Push config from local dir <%s> to etcd dir <%s>\n", root, etcdRoot)
checkGitVersion()
// check if root is symbolic link and convert it
if r, err := filepath.EvalSymlinks(root); err == nil {
fmt.Printf("convert symbolic link root %s to %s\n", root, r)
root = r
}
// cd to repo or fatal
cdRepo(root)
// get local repo root
repoRoot := runSingleCmdOrFatal("git rev-parse --show-toplevel")
fmt.Printf("local repo root: %s\n", repoRoot)
// get repo name
repo := path.Base(repoRoot)
fmt.Printf("using repo: %s\n", repo)
// get path to git root
pathToRepo, err := filepath.Rel(repoRoot, root)
if err != nil {
log.Fatalf("unable to find relative path from <%s> to <%s>, err %s", repoRoot, root, err)
}
// get branch name
branch := runSingleCmdOrFatal("git rev-parse --abbrev-ref HEAD")
// TODO: may need to relax this constrain.
/*
if branch != allowedPusherBranch {
log.Fatalf("only %s branch is allowed to push, whereas you currently are in %s", allowedPusherBranch, branch)
}
*/
fmt.Printf("using branch: %s\n", branch)
// get commit hash
commitHash := runSingleCmdOrFatal("git rev-parse HEAD")
fmt.Printf("using commit: %s\n", commitHash)
// get timestamp, committer email and subject of a commit
info := runSingleCmdOrFatal("git show --quiet --pretty=%at:%ce:%s " + commitHash)
fmt.Printf("commit timestamp:email:subject: %s\n", info)
tokens := strings.Split(info, ":")
if len(tokens) < 3 {
log.Fatalf("commit info is not in the timestamp:email:subject format:%s\n", info)
}
tsstr, email, subject := tokens[0], tokens[1], strings.Join(tokens[2:], ":")
// convert tsstr to timestamp
timestamp, _ := strconv.ParseInt(tsstr, 10, 64)
// set etcd
remoteRoot := strings.Trim(etcdRoot, " \t")
if remoteRoot == "" {
log.Fatalf("etcdRoot is empty\n")
}
infoPath := fmt.Sprintf(infoPrefix, remoteRoot)
// check config root on etcd
resp, err := etcdConn.Get(remoteRoot, true, false)
if err != nil {
log.Fatalf("error testing remoteRoot, %s: %s\n", remoteRoot, err)
}
log.Infof("remoteRoot %s verified\n", remoteRoot)
// default etcdLastCommit is current repo newest commit
prevCFG := &config.ConfigInfo{}
etcdLastCommit := []byte(runSingleCmdOrFatal("git rev-list --max-parents=0 HEAD"))
etcdHasCommit := false
log.Infof("reading last pushed information for %s", infoPath)
resp, err = etcdConn.Get(infoPath, true, false)
if err == nil {
jerr := json.Unmarshal([]byte(resp.Node.Value), prevCFG)
if jerr == nil && prevCFG.Version != "" {
// read previous commit
etcdLastCommit = []byte(prevCFG.Version)
etcdHasCommit = true
}
} else {
// previos config is empty or invalid
log.Infof("previous configInfo doesn't exist")
}
// TODO: empty etcdLastCommit cause diff-tree to only print files of current commit.
filestr := runSingleCmdOrFatal(fmt.Sprintf("git diff-tree --no-commit-id --name-status -r %s %s %s", etcdLastCommit, commitHash, root))
// filter out files that are descendents of the rootpath
modFiles := filterRelatedFiles(root, repoRoot, filestr)
if len(*modFiles) == 0 {
promptUser("Remote repo seems to be already up-to-date "+
"(remote commit %s vs local commit %s). Do you want to continue?",
etcdLastCommit, commitHash)
} else {
fmt.Println("This push will notify the following file changes:")
for _, m := range *modFiles {
fmt.Printf("%+v\n", m)
}
fmt.Println("End of notify changes.")
}
if etcdHasCommit {
fmt.Printf("Remote commit: %s\n",
runSingleCmdOrFatal("git log --format=%h:%s(%aN) -n 1 "+string(etcdLastCommit)))
}
fmt.Printf("Local commit: %s\n",
runSingleCmdOrFatal("git log --format=%h:%s(%aN) -n 1 "+commitHash))
fmt.Printf("Total commit(s) to be pushed: %s\n",
runSingleCmdOrFatal("git rev-list --count "+commitHash+" ^"+string(etcdLastCommit)))
fmt.Printf("Files changed between remote/local commits:\n%s\n",
runSingleCmdOrFatal("git diff --name-only "+string(etcdLastCommit)+" "+commitHash+" "+root))
promptUser("Ready to push?")
// walk through the repo and save content
content := make(map[string]fileInfo)
// since Walk will pass in absolute path, we are going to take
// the root out from the beginning
sz := len(root)
walkFunc := func(path string, info os.FileInfo, err error) error {
if err != nil {
log.Fatalf("Unable to traverse file %s: %s", path, err)
}
// TODO: we no need to scan all the files in the localRepo. Just check the files which are tracked by git: git ls-files
// ignore .xxx file
base := filepath.Base(path)
// skip hidden dir, e.g. .git
if string(base[0]) == "." {
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
bs := []byte{}
// for dir we set "" content
if !info.IsDir() {
bs, err = ioutil.ReadFile(path)
if err != nil {
log.Fatalf("error reading file %s: %s\n", path, err)
}
bs = bytes.TrimRight(bs, "\n\r")
}
// skip root
if path == root {
return nil
}
// remove root path
path = path[sz:]
content[path] = fileInfo{isDir: info.IsDir(), content: bs}
return nil
}
if err = filepath.Walk(root, walkFunc); err != nil {
log.Fatalf("error when traversing %s: %s\n", root, err)
}
configInfo := config.ConfigInfo{
Repo: repo,
Branch: branch,
Version: commitHash,
Commit: config.CommitInfo{
TimeStamp: timestamp,
CommitterEmail: email,
Subject: subject,
},
ModFiles: *modFiles,
PathToRepo: pathToRepo,
}
jsonBytes, err := json.Marshal(configInfo)
if err != nil {
log.Fatalf("error when marshal configInfo, err: %s", err)
}
fmt.Printf("ConfigInfo json: %s\n", string(jsonBytes[:len(jsonBytes)]))
// Get previous pusher information from root node for sanity check.
if !etcdHasCommit {
promptUser("There is no existing commit in remote tree. It could because " +
"you're pushing to a clean tree. Do you want to continue?")
} else {
// Sanity check of pusher info.
if configInfo.Repo != prevCFG.Repo {
promptUser("Repo to be pushed <%s> != remote repo name <%s>. Continue?",
configInfo.Repo, prevCFG.Repo)
}
if configInfo.Branch != prevCFG.Branch {
promptUser("Branch to be pushed <%s> != remote branch <%s>. Continue?",
configInfo.Branch, prevCFG.Branch)
}
if configInfo.PathToRepo != prevCFG.PathToRepo {
promptUser("Path to repo <%s> != remote path <%s>. Continue?",
configInfo.PathToRepo, prevCFG.PathToRepo)
}
}
keys := make([]string, len(content))
// sort content's keys
i := 0
for k := range content {
keys[i] = k
i++
}
sort.Strings(keys)
// set etcd content.
for _, k := range keys {
fileP := filepath.Join(remoteRoot, k)
fmt.Printf("creating or setting %s\n", fileP)
if content[k].isDir {
if _, err := etcdConn.Get(fileP, true, false); err != nil {
// dir doesn't exist
resp, err = etcdConn.SetDir(fileP, 0)
}
} else {
resp, err = etcdConn.Set(fileP, string(content[k].content), 0)
}
if err != nil {
log.Fatalf("error when setting znode >%s(%s + %s)<. Config server will be inconsistent: %s",
fileP, remoteRoot, k, err)
}
}
// go over deletes in commit
for _, mod := range *modFiles {
if strings.ToLower(mod.Op) != "d" {
continue
}
// it's a delete
// Find the relative path to etcd root.
fileP := filepath.Join(remoteRoot, mod.Path)
fmt.Printf("deleting %s (%s + %s)\n", fileP, remoteRoot, mod.Path)
_, err = etcdConn.Delete(fileP, true)
if err != nil {
log.Errorf("error deleting file >%s<. Will continue. error: %s\n", fileP, err)
}
// since git uses a file as a commit unit, there is nothing to do with folder.
// what we are going to do is to delete each fileP which is modified with "d" and its corresponding folder.
// If we still have children under that folder, deleting the folder will fail but we do not care.
pDir := filepath.Join(remoteRoot, path.Dir(mod.Path))
_, err = etcdConn.DeleteDir(pDir)
// In normal case, we should get an error.
// so just logging, if we have no error here.
if err == nil {
log.Errorf("error deleting dir >%s<.\n", pDir)
}
}
// touch the root node with commit info
_, err = etcdConn.Set(infoPath, string(jsonBytes), 0)
if err != nil {
log.Fatalf("error setting remoteRoot >%s<: %s\n", remoteRoot, err)
}
fmt.Printf("All done\n")
}
func etcdClientSetUp(t *testing.T, rawClient *etcd.Client) map[string]interface{} {
var err error
if _, err = rawClient.SetDir("test", 0); err != nil {
t.Fatal(err)
}
if _, err = rawClient.Set("test/index", "1", 0); err != nil {
t.Fatal(err)
}
if _, err = rawClient.SetDir("test/values", 0); err != nil {
t.Fatal(err)
}
if _, err = rawClient.Set("test/values/a", "aye", 0); err != nil {
t.Fatal(err)
}
if _, err = rawClient.Set("test/values/b", "bee", 0); err != nil {
t.Fatal(err)
}
want := make(map[string]interface{})
wantValues := make(map[string]interface{})
wantValues["a"] = "aye"
wantValues["b"] = "bee"
want["values"] = wantValues
want["index"] = "1"
return want
}
func EtcdMkDir(client *etcd.Client, path string, ttl uint64) error {
_, err := client.SetDir(path, ttl)
if err != nil {
return err
}
return nil
}
func setDirCommandFunc(cmd *cobra.Command, args []string, client *etcd.Client) (*etcd.Response, error) {
if len(args) == 0 {
return nil, errors.New("Key required")
}
key := args[0]
ttl := ttlFlag
return client.SetDir(key, uint64(ttl))
}
func RegisterService(client *etcd.Client, service services.Service, ttl time.Duration) {
keyPrefix := fmt.Sprintf("/services/%s/%s/", service.Service, service.Id)
var attributes map[string]string = make(map[string]string)
attributes["address"] = service.Address
for key, value := range service.Labels {
if key == "address" {
fmt.Println("WARNING: overriding address(%s) with label: %s", service.Address, value)
}
attributes[key] = value
}
// create service/id dir with ttl
client.SetDir(keyPrefix, uint64(ttl.Seconds()))
for key, value := range attributes {
_, err := client.Set(keyPrefix+key, value, 0)
if err != nil {
fmt.Println(err)
// abort current registration
return
}
}
}
func TestNodeFs(t *testing.T) {
g := Goblin(t)
g.Describe("File", func() {
var etcd *etcdm.Client
var fs testEtcdFsMount
g.Before(func() {
etcd = etcdm.NewClient([]string{testEtcdEndpoint})
})
g.BeforeEach(func() {
etcd.RawDelete("/test", true, true)
etcd.SetDir("/test", 0)
fs = NewTestEtcdFsMount()
})
g.AfterEach(func() {
fs.Unmount()
})
g.Describe("Open", func() {
g.It("Should be supported", func() {
if _, e := etcd.Set("/test/foo", "bar", 0); e != nil {
g.Fail(e)
}
file, err := os.Open(fs.Path() + "/test/foo")
if err != nil {
g.Fail(err)
}
file.Close()
})
})
g.Describe("Create", func() {
g.It("Should be supported", func() {
file, err := os.Create(fs.Path() + "/test/bar")
if err != nil {
g.Fail(err)
}
file.Close()
if _, er := etcd.Get("/test/bar", false, false); er != nil {
g.Fail(er)
}
})
})
g.Describe("Delete", func() {
g.It("Should be supported", func() {
etcd.Set("/test/barfoo", "lala", 0)
err := os.Remove(fs.Path() + "/test/barfoo")
if err != nil {
g.Fail(err)
}
if _, er := etcd.Get("/test/barfoo", false, false); er == nil {
g.Fail("The key [/test/barfoo] should not exist")
}
})
})
g.Describe("Read", func() {
g.It("Should be supported", func() {
etcd.Set("/test/bar", "foo", 0)
data, err := ioutil.ReadFile(fs.Path() + "/test/bar")
if err != nil {
g.Fail(err)
}
g.Assert(string(data)).Equal("foo")
})
})
g.Describe("Write", func() {
g.It("Should be supported", func() {
if err := ioutil.WriteFile(fs.Path()+"/test/foobar", []byte("hello world"), 0666); err != nil {
g.Fail(err)
}
res, err := etcd.Get("/test/foobar", false, false)
if err != nil {
g.Fail(err)
}
g.Assert(res.Node.Value).Equal("hello world")
})
})
})
}