// Pusher push the localRepo to etcd.
func Pusher(etcdConn *etcd.Client, root, etcdRoot string) {
fmt.Printf("Push config from local dir <%s> to etcd dir <%s>\n", root, etcdRoot)
checkGitVersion()
// check if root is symbolic link and convert it
if r, err := filepath.EvalSymlinks(root); err == nil {
fmt.Printf("convert symbolic link root %s to %s\n", root, r)
root = r
}
// cd to repo or fatal
cdRepo(root)
// get local repo root
repoRoot := runSingleCmdOrFatal("git rev-parse --show-toplevel")
fmt.Printf("local repo root: %s\n", repoRoot)
// get repo name
repo := path.Base(repoRoot)
fmt.Printf("using repo: %s\n", repo)
// get path to git root
pathToRepo, err := filepath.Rel(repoRoot, root)
if err != nil {
log.Fatalf("unable to find relative path from <%s> to <%s>, err %s", repoRoot, root, err)
}
// get branch name
branch := runSingleCmdOrFatal("git rev-parse --abbrev-ref HEAD")
// TODO: may need to relax this constrain.
/*
if branch != allowedPusherBranch {
log.Fatalf("only %s branch is allowed to push, whereas you currently are in %s", allowedPusherBranch, branch)
}
*/
fmt.Printf("using branch: %s\n", branch)
// get commit hash
commitHash := runSingleCmdOrFatal("git rev-parse HEAD")
fmt.Printf("using commit: %s\n", commitHash)
// get timestamp, committer email and subject of a commit
info := runSingleCmdOrFatal("git show --quiet --pretty=%at:%ce:%s " + commitHash)
fmt.Printf("commit timestamp:email:subject: %s\n", info)
tokens := strings.Split(info, ":")
if len(tokens) < 3 {
log.Fatalf("commit info is not in the timestamp:email:subject format:%s\n", info)
}
tsstr, email, subject := tokens[0], tokens[1], strings.Join(tokens[2:], ":")
// convert tsstr to timestamp
timestamp, _ := strconv.ParseInt(tsstr, 10, 64)
// set etcd
remoteRoot := strings.Trim(etcdRoot, " \t")
if remoteRoot == "" {
log.Fatalf("etcdRoot is empty\n")
}
infoPath := fmt.Sprintf(infoPrefix, remoteRoot)
// check config root on etcd
resp, err := etcdConn.Get(remoteRoot, true, false)
if err != nil {
log.Fatalf("error testing remoteRoot, %s: %s\n", remoteRoot, err)
}
log.Infof("remoteRoot %s verified\n", remoteRoot)
// default etcdLastCommit is current repo newest commit
prevCFG := &config.ConfigInfo{}
etcdLastCommit := []byte(runSingleCmdOrFatal("git rev-list --max-parents=0 HEAD"))
etcdHasCommit := false
log.Infof("reading last pushed information for %s", infoPath)
resp, err = etcdConn.Get(infoPath, true, false)
if err == nil {
jerr := json.Unmarshal([]byte(resp.Node.Value), prevCFG)
if jerr == nil && prevCFG.Version != "" {
// read previous commit
etcdLastCommit = []byte(prevCFG.Version)
etcdHasCommit = true
}
} else {
// previos config is empty or invalid
log.Infof("previous configInfo doesn't exist")
}
// TODO: empty etcdLastCommit cause diff-tree to only print files of current commit.
filestr := runSingleCmdOrFatal(fmt.Sprintf("git diff-tree --no-commit-id --name-status -r %s %s %s", etcdLastCommit, commitHash, root))
// filter out files that are descendents of the rootpath
modFiles := filterRelatedFiles(root, repoRoot, filestr)
if len(*modFiles) == 0 {
promptUser("Remote repo seems to be already up-to-date "+
"(remote commit %s vs local commit %s). Do you want to continue?",
etcdLastCommit, commitHash)
} else {
fmt.Println("This push will notify the following file changes:")
for _, m := range *modFiles {
fmt.Printf("%+v\n", m)
}
fmt.Println("End of notify changes.")
}
if etcdHasCommit {
fmt.Printf("Remote commit: %s\n",
runSingleCmdOrFatal("git log --format=%h:%s(%aN) -n 1 "+string(etcdLastCommit)))
}
fmt.Printf("Local commit: %s\n",
runSingleCmdOrFatal("git log --format=%h:%s(%aN) -n 1 "+commitHash))
fmt.Printf("Total commit(s) to be pushed: %s\n",
runSingleCmdOrFatal("git rev-list --count "+commitHash+" ^"+string(etcdLastCommit)))
fmt.Printf("Files changed between remote/local commits:\n%s\n",
runSingleCmdOrFatal("git diff --name-only "+string(etcdLastCommit)+" "+commitHash+" "+root))
promptUser("Ready to push?")
// walk through the repo and save content
content := make(map[string]fileInfo)
// since Walk will pass in absolute path, we are going to take
// the root out from the beginning
sz := len(root)
walkFunc := func(path string, info os.FileInfo, err error) error {
if err != nil {
log.Fatalf("Unable to traverse file %s: %s", path, err)
}
// TODO: we no need to scan all the files in the localRepo. Just check the files which are tracked by git: git ls-files
// ignore .xxx file
base := filepath.Base(path)
// skip hidden dir, e.g. .git
if string(base[0]) == "." {
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
bs := []byte{}
// for dir we set "" content
if !info.IsDir() {
bs, err = ioutil.ReadFile(path)
if err != nil {
log.Fatalf("error reading file %s: %s\n", path, err)
}
bs = bytes.TrimRight(bs, "\n\r")
}
// skip root
if path == root {
return nil
}
// remove root path
path = path[sz:]
content[path] = fileInfo{isDir: info.IsDir(), content: bs}
return nil
}
if err = filepath.Walk(root, walkFunc); err != nil {
log.Fatalf("error when traversing %s: %s\n", root, err)
}
configInfo := config.ConfigInfo{
Repo: repo,
Branch: branch,
Version: commitHash,
Commit: config.CommitInfo{
TimeStamp: timestamp,
CommitterEmail: email,
Subject: subject,
},
ModFiles: *modFiles,
PathToRepo: pathToRepo,
}
jsonBytes, err := json.Marshal(configInfo)
if err != nil {
log.Fatalf("error when marshal configInfo, err: %s", err)
}
fmt.Printf("ConfigInfo json: %s\n", string(jsonBytes[:len(jsonBytes)]))
// Get previous pusher information from root node for sanity check.
if !etcdHasCommit {
promptUser("There is no existing commit in remote tree. It could because " +
"you're pushing to a clean tree. Do you want to continue?")
} else {
// Sanity check of pusher info.
if configInfo.Repo != prevCFG.Repo {
promptUser("Repo to be pushed <%s> != remote repo name <%s>. Continue?",
configInfo.Repo, prevCFG.Repo)
}
if configInfo.Branch != prevCFG.Branch {
promptUser("Branch to be pushed <%s> != remote branch <%s>. Continue?",
configInfo.Branch, prevCFG.Branch)
}
if configInfo.PathToRepo != prevCFG.PathToRepo {
promptUser("Path to repo <%s> != remote path <%s>. Continue?",
configInfo.PathToRepo, prevCFG.PathToRepo)
}
}
keys := make([]string, len(content))
// sort content's keys
i := 0
for k := range content {
keys[i] = k
i++
}
sort.Strings(keys)
// set etcd content.
for _, k := range keys {
fileP := filepath.Join(remoteRoot, k)
fmt.Printf("creating or setting %s\n", fileP)
if content[k].isDir {
if _, err := etcdConn.Get(fileP, true, false); err != nil {
// dir doesn't exist
resp, err = etcdConn.SetDir(fileP, 0)
}
} else {
resp, err = etcdConn.Set(fileP, string(content[k].content), 0)
}
if err != nil {
log.Fatalf("error when setting znode >%s(%s + %s)<. Config server will be inconsistent: %s",
fileP, remoteRoot, k, err)
}
}
// go over deletes in commit
for _, mod := range *modFiles {
if strings.ToLower(mod.Op) != "d" {
continue
}
// it's a delete
// Find the relative path to etcd root.
fileP := filepath.Join(remoteRoot, mod.Path)
fmt.Printf("deleting %s (%s + %s)\n", fileP, remoteRoot, mod.Path)
_, err = etcdConn.Delete(fileP, true)
if err != nil {
log.Errorf("error deleting file >%s<. Will continue. error: %s\n", fileP, err)
}
// since git uses a file as a commit unit, there is nothing to do with folder.
// what we are going to do is to delete each fileP which is modified with "d" and its corresponding folder.
// If we still have children under that folder, deleting the folder will fail but we do not care.
pDir := filepath.Join(remoteRoot, path.Dir(mod.Path))
_, err = etcdConn.DeleteDir(pDir)
// In normal case, we should get an error.
// so just logging, if we have no error here.
if err == nil {
log.Errorf("error deleting dir >%s<.\n", pDir)
}
}
// touch the root node with commit info
_, err = etcdConn.Set(infoPath, string(jsonBytes), 0)
if err != nil {
log.Fatalf("error setting remoteRoot >%s<: %s\n", remoteRoot, err)
}
fmt.Printf("All done\n")
}
func cleanup(r *bytes.Buffer, e *etcd.Client, ep *testProcess, dp *testProcess) {
if r != nil {
log.Debug("Writing report")
rpath := testDir + "/report.txt"
if err := ioutil.WriteFile(rpath, r.Bytes(), 0644); err != nil {
log.WithFields(log.Fields{
"error": err,
"func": "ioutil.WriteFile",
"path": rpath,
}).Warning("Could not write report")
}
}
if dp != nil {
log.Debug("Exiting cdhcpd")
_ = dp.finish()
time.Sleep(time.Second)
}
if e != nil {
log.Debug("Clearing test data")
if _, err := e.Delete("/lochness", true); err != nil {
log.WithFields(log.Fields{
"error": err,
"func": "etcd.Delete",
}).Warning("Could not clear test-created data from etcd")
}
time.Sleep(time.Second)
}
if ep != nil {
log.Debug("Exiting etcd")
_ = ep.finish()
}
log.Info("Done")
}
func RemoveService(client *etcd.Client, s *Service) error {
basekey := appendPrefix(fmt.Sprintf("/beacon/registry/%s/haproxy/%s",
s.Cluster, s.Proto), s.Prefix)
var key string
if s.Proto == "tcp" {
key = fmt.Sprintf("%s/%s", basekey, s.Name)
} else {
key = fmt.Sprintf("%s/%s/backends/%s", basekey, s.Name, s.Backend)
}
log.WithFields(log.Fields{
"key": key,
}).Debugln("start to delete a service key.")
if _, err := client.Delete(key, true); err != nil {
return err
}
log.WithFields(log.Fields{
"name": s.Name,
"backend": s.Backend,
"proto": s.Proto,
}).Infoln("service unregistered.")
return nil
}
func (etcd *ETCDClusterRunner) deleteDir(client *etcdclient.Client, dir string) {
responses, err := client.Get(dir)
Ω(err).ShouldNot(HaveOccured())
for _, response := range responses {
if response.Key != "/_etcd" {
if response.Dir == true {
etcd.deleteDir(client, response.Key)
} else {
_, err := client.Delete(response.Key)
Ω(err).ShouldNot(HaveOccured())
}
}
}
}
func TryOccupyTask(client *etcd.Client, name string, taskID uint64, connection string) (bool, error) {
_, err := client.Create(TaskHealthyPath(name, taskID), "health", 3)
if err != nil {
if strings.Contains(err.Error(), "Key already exists") {
return false, nil
}
return false, err
}
idStr := strconv.FormatUint(taskID, 10)
client.Delete(FreeTaskPath(name, idStr), false)
_, err = client.Set(TaskMasterPath(name, taskID), connection, 0)
if err != nil {
return false, err
}
return true, nil
}
func (ec *EtcdCoordinator) parseCommand(c *etcd.Client, resp *etcd.Response) metafora.Command {
if strings.HasSuffix(resp.Node.Key, MetadataKey) {
// Skip metadata marker
return nil
}
const recurse = false
if _, err := c.Delete(resp.Node.Key, recurse); err != nil {
metafora.Errorf("Error deleting handled command %s: %v", resp.Node.Key, err)
}
cmd, err := metafora.UnmarshalCommand([]byte(resp.Node.Value))
if err != nil {
metafora.Errorf("Invalid command %s: %v", resp.Node.Key, err)
return nil
}
return cmd
}
func (locker *EtcdLocker) acquire(client *etcd.Client, key string, ttl uint64, wait bool) (Lock, error) {
hasLock := false
key = addPrefix(key)
lock, err := addLockDirChild(client, key)
if err != nil {
return nil, errgo.Mask(err)
}
for !hasLock {
res, err := client.Get(key, true, true)
if err != nil {
return nil, errgo.Mask(err)
}
if len(res.Node.Nodes) > 1 {
sort.Sort(res.Node.Nodes)
if res.Node.Nodes[0].CreatedIndex != lock.Node.CreatedIndex {
if !wait {
client.Delete(lock.Node.Key, false)
return nil, &Error{res.Node.Nodes[0].Value}
} else {
err = locker.Wait(lock.Node.Key)
if err != nil {
return nil, errgo.Mask(err)
}
}
} else {
// if the first index is the current one, it's our turn to lock the key
hasLock = true
}
} else {
// If there are only 1 node, it's our, lock is acquired
hasLock = true
}
}
// If we get the lock, set the ttl and return it
_, err = client.Update(lock.Node.Key, lock.Node.Value, ttl)
if err != nil {
return nil, errgo.Mask(err)
}
return &EtcdLock{client, lock.Node.Key, lock.Node.CreatedIndex}, nil
}
func etcd_GetDeadKeyDelete() {
info := config.GetServerInfo()
log.Printf("Dead info : %s", info.AccessString)
//key := "SP/Dead"
key := fmt.Sprintf("%s/Dead", info.AppName)
log.Printf("Dead list info key : %s\n", key)
var client *etcd.Client = nil
client = etcd.NewClient(info.AccessString)
resp, err := client.Delete(key, true)
if err != nil {
log.Fatal(err)
}
log.Printf(">>>Dead Key information>>>%s %s", resp.Node.Key, resp.Node.Value)
}
func (s *SchedulerServer) fetchFrameworkID(client *etcd.Client) (*mesos.FrameworkID, error) {
if s.failoverTimeout > 0 {
if response, err := client.Get(meta.FrameworkIDKey, false, false); err != nil {
if !etcdutil.IsEtcdNotFound(err) {
return nil, fmt.Errorf("unexpected failure attempting to load framework ID from etcd: %v", err)
}
log.V(1).Infof("did not find framework ID in etcd")
} else if response.Node.Value != "" {
log.Infof("configuring FrameworkInfo with Id found in etcd: '%s'", response.Node.Value)
return mutil.NewFrameworkID(response.Node.Value), nil
}
} else {
//TODO(jdef) this seems like a totally hackish way to clean up the framework ID
if _, err := client.Delete(meta.FrameworkIDKey, true); err != nil {
if !etcdutil.IsEtcdNotFound(err) {
return nil, fmt.Errorf("failed to delete framework ID from etcd: %v", err)
}
log.V(1).Infof("nothing to delete: did not find framework ID in etcd")
}
}
return nil, nil
}
func cleanup(c *etcd.Client, key string) {
if _, err := c.Delete(key, true); err != nil && !isKeyNotFound(err) {
fmt.Printf("Unable to delete key '%s': %+v", key, err)
}
}
func etcdClientTearDown(t *testing.T, rawClient *etcd.Client) {
if _, err := rawClient.Delete("test", true); err != nil {
t.Fatal(err)
}
}
func removeDNS(record string, etcdClient *etcd.Client) error {
log.Printf("Removing %s from DNS", record)
_, err := etcdClient.Delete(skymsg.Path(record), true)
return err
}
func deleteService(client *etcd.Client, service string, addr string) {
client.Delete(fmt.Sprintf("/services/%s/%s", service, addr))
}
