Example #1
1
File: main.go Project: csigo/config
// Pusher push the localRepo to etcd.
func Pusher(etcdConn *etcd.Client, root, etcdRoot string) {
	fmt.Printf("Push config from local dir <%s> to etcd dir <%s>\n", root, etcdRoot)

	checkGitVersion()

	// check if root is symbolic link and convert it
	if r, err := filepath.EvalSymlinks(root); err == nil {
		fmt.Printf("convert symbolic link root %s to %s\n", root, r)
		root = r
	}

	// cd to repo or fatal
	cdRepo(root)

	// get local repo root
	repoRoot := runSingleCmdOrFatal("git rev-parse --show-toplevel")
	fmt.Printf("local repo root: %s\n", repoRoot)

	// get repo name
	repo := path.Base(repoRoot)
	fmt.Printf("using repo: %s\n", repo)

	// get path to git root
	pathToRepo, err := filepath.Rel(repoRoot, root)
	if err != nil {
		log.Fatalf("unable to find relative path from <%s> to <%s>, err %s", repoRoot, root, err)
	}

	// get branch name
	branch := runSingleCmdOrFatal("git rev-parse --abbrev-ref HEAD")
	// TODO: may need to relax this constrain.
	/*
		if branch != allowedPusherBranch {
			log.Fatalf("only %s branch is allowed to push, whereas you currently are in %s", allowedPusherBranch, branch)
		}
	*/
	fmt.Printf("using branch: %s\n", branch)

	// get commit hash
	commitHash := runSingleCmdOrFatal("git rev-parse HEAD")
	fmt.Printf("using commit: %s\n", commitHash)

	// get timestamp, committer email and subject of a commit
	info := runSingleCmdOrFatal("git show --quiet --pretty=%at:%ce:%s " + commitHash)
	fmt.Printf("commit timestamp:email:subject: %s\n", info)
	tokens := strings.Split(info, ":")
	if len(tokens) < 3 {
		log.Fatalf("commit info is not in the timestamp:email:subject format:%s\n", info)
	}

	tsstr, email, subject := tokens[0], tokens[1], strings.Join(tokens[2:], ":")
	// convert tsstr to timestamp
	timestamp, _ := strconv.ParseInt(tsstr, 10, 64)

	// set etcd
	remoteRoot := strings.Trim(etcdRoot, " \t")
	if remoteRoot == "" {
		log.Fatalf("etcdRoot is empty\n")
	}
	infoPath := fmt.Sprintf(infoPrefix, remoteRoot)

	// check config root on etcd
	resp, err := etcdConn.Get(remoteRoot, true, false)
	if err != nil {
		log.Fatalf("error testing remoteRoot, %s: %s\n", remoteRoot, err)
	}
	log.Infof("remoteRoot %s verified\n", remoteRoot)

	// default etcdLastCommit is current repo newest commit
	prevCFG := &config.ConfigInfo{}
	etcdLastCommit := []byte(runSingleCmdOrFatal("git rev-list --max-parents=0 HEAD"))
	etcdHasCommit := false
	log.Infof("reading last pushed information for %s", infoPath)
	resp, err = etcdConn.Get(infoPath, true, false)
	if err == nil {
		jerr := json.Unmarshal([]byte(resp.Node.Value), prevCFG)
		if jerr == nil && prevCFG.Version != "" {
			// read previous commit
			etcdLastCommit = []byte(prevCFG.Version)
			etcdHasCommit = true
		}
	} else {
		// previos config is empty or invalid
		log.Infof("previous configInfo doesn't exist")
	}

	// TODO: empty etcdLastCommit cause diff-tree to only print files of current commit.
	filestr := runSingleCmdOrFatal(fmt.Sprintf("git diff-tree --no-commit-id --name-status -r %s %s %s", etcdLastCommit, commitHash, root))
	// filter out files that are descendents of the rootpath
	modFiles := filterRelatedFiles(root, repoRoot, filestr)
	if len(*modFiles) == 0 {
		promptUser("Remote repo seems to be already up-to-date "+
			"(remote commit %s vs local commit %s). Do you want to continue?",
			etcdLastCommit, commitHash)
	} else {
		fmt.Println("This push will notify the following file changes:")
		for _, m := range *modFiles {
			fmt.Printf("%+v\n", m)
		}
		fmt.Println("End of notify changes.")
	}

	if etcdHasCommit {
		fmt.Printf("Remote commit: %s\n",
			runSingleCmdOrFatal("git log --format=%h:%s(%aN) -n 1 "+string(etcdLastCommit)))
	}
	fmt.Printf("Local commit:  %s\n",
		runSingleCmdOrFatal("git log --format=%h:%s(%aN) -n 1 "+commitHash))
	fmt.Printf("Total commit(s) to be pushed: %s\n",
		runSingleCmdOrFatal("git rev-list --count "+commitHash+" ^"+string(etcdLastCommit)))
	fmt.Printf("Files changed between remote/local commits:\n%s\n",
		runSingleCmdOrFatal("git diff --name-only "+string(etcdLastCommit)+" "+commitHash+" "+root))
	promptUser("Ready to push?")

	// walk through the repo and save content
	content := make(map[string]fileInfo)

	// since Walk will pass in absolute path, we are going to take
	// the root out from the beginning
	sz := len(root)

	walkFunc := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			log.Fatalf("Unable to traverse file %s: %s", path, err)
		}

		// TODO: we no need to scan all the files in the localRepo. Just check the files which are tracked by git: git ls-files
		// ignore .xxx file
		base := filepath.Base(path)
		// skip hidden dir, e.g. .git
		if string(base[0]) == "." {
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		bs := []byte{}
		// for dir we set "" content
		if !info.IsDir() {
			bs, err = ioutil.ReadFile(path)
			if err != nil {
				log.Fatalf("error reading file %s: %s\n", path, err)
			}
			bs = bytes.TrimRight(bs, "\n\r")
		}

		// skip root
		if path == root {
			return nil
		}
		// remove root path
		path = path[sz:]
		content[path] = fileInfo{isDir: info.IsDir(), content: bs}
		return nil
	}

	if err = filepath.Walk(root, walkFunc); err != nil {
		log.Fatalf("error when traversing %s: %s\n", root, err)
	}

	configInfo := config.ConfigInfo{
		Repo:    repo,
		Branch:  branch,
		Version: commitHash,
		Commit: config.CommitInfo{
			TimeStamp:      timestamp,
			CommitterEmail: email,
			Subject:        subject,
		},
		ModFiles:   *modFiles,
		PathToRepo: pathToRepo,
	}

	jsonBytes, err := json.Marshal(configInfo)
	if err != nil {
		log.Fatalf("error when marshal configInfo, err: %s", err)
	}
	fmt.Printf("ConfigInfo json: %s\n", string(jsonBytes[:len(jsonBytes)]))

	// Get previous pusher information from root node for sanity check.
	if !etcdHasCommit {
		promptUser("There is no existing commit in remote tree. It could because " +
			"you're pushing to a clean tree. Do you want to continue?")
	} else {
		// Sanity check of pusher info.
		if configInfo.Repo != prevCFG.Repo {
			promptUser("Repo to be pushed <%s> != remote repo name <%s>. Continue?",
				configInfo.Repo, prevCFG.Repo)
		}
		if configInfo.Branch != prevCFG.Branch {
			promptUser("Branch to be pushed <%s> != remote branch <%s>. Continue?",
				configInfo.Branch, prevCFG.Branch)
		}
		if configInfo.PathToRepo != prevCFG.PathToRepo {
			promptUser("Path to repo <%s> != remote path <%s>. Continue?",
				configInfo.PathToRepo, prevCFG.PathToRepo)
		}
	}

	keys := make([]string, len(content))
	// sort content's keys
	i := 0
	for k := range content {
		keys[i] = k
		i++
	}
	sort.Strings(keys)

	// set etcd content.
	for _, k := range keys {
		fileP := filepath.Join(remoteRoot, k)
		fmt.Printf("creating or setting %s\n", fileP)
		if content[k].isDir {
			if _, err := etcdConn.Get(fileP, true, false); err != nil {
				// dir doesn't exist
				resp, err = etcdConn.SetDir(fileP, 0)
			}
		} else {
			resp, err = etcdConn.Set(fileP, string(content[k].content), 0)
		}
		if err != nil {
			log.Fatalf("error when setting znode >%s(%s + %s)<. Config server will be inconsistent: %s",
				fileP, remoteRoot, k, err)
		}
	}

	// go over deletes in commit
	for _, mod := range *modFiles {
		if strings.ToLower(mod.Op) != "d" {
			continue
		}
		// it's a delete
		// Find the relative path to etcd root.
		fileP := filepath.Join(remoteRoot, mod.Path)
		fmt.Printf("deleting %s (%s + %s)\n", fileP, remoteRoot, mod.Path)
		_, err = etcdConn.Delete(fileP, true)
		if err != nil {
			log.Errorf("error deleting file >%s<. Will continue. error: %s\n", fileP, err)
		}
		// since git uses a file as  a commit unit, there is nothing to do with folder.
		// what we are going to do is to delete each fileP which is modified with "d" and its corresponding folder.
		// If we still have children under that folder, deleting the folder will fail but we do not care.
		pDir := filepath.Join(remoteRoot, path.Dir(mod.Path))
		_, err = etcdConn.DeleteDir(pDir)
		// In normal case, we should get an error.
		// so just logging, if we have no error here.
		if err == nil {
			log.Errorf("error deleting dir >%s<.\n", pDir)
		}
	}

	// touch the root node with commit info
	_, err = etcdConn.Set(infoPath, string(jsonBytes), 0)
	if err != nil {
		log.Fatalf("error setting remoteRoot >%s<: %s\n", remoteRoot, err)
	}
	fmt.Printf("All done\n")
}
Example #2
0
func cleanup(r *bytes.Buffer, e *etcd.Client, ep *testProcess, dp *testProcess) {
	if r != nil {
		log.Debug("Writing report")
		rpath := testDir + "/report.txt"
		if err := ioutil.WriteFile(rpath, r.Bytes(), 0644); err != nil {
			log.WithFields(log.Fields{
				"error": err,
				"func":  "ioutil.WriteFile",
				"path":  rpath,
			}).Warning("Could not write report")
		}
	}
	if dp != nil {
		log.Debug("Exiting cdhcpd")
		_ = dp.finish()
		time.Sleep(time.Second)
	}
	if e != nil {
		log.Debug("Clearing test data")
		if _, err := e.Delete("/lochness", true); err != nil {
			log.WithFields(log.Fields{
				"error": err,
				"func":  "etcd.Delete",
			}).Warning("Could not clear test-created data from etcd")
		}
		time.Sleep(time.Second)
	}
	if ep != nil {
		log.Debug("Exiting etcd")
		_ = ep.finish()
	}
	log.Info("Done")
}
Example #3
0
func RemoveService(client *etcd.Client, s *Service) error {
	basekey := appendPrefix(fmt.Sprintf("/beacon/registry/%s/haproxy/%s",
		s.Cluster, s.Proto), s.Prefix)

	var key string
	if s.Proto == "tcp" {
		key = fmt.Sprintf("%s/%s", basekey, s.Name)
	} else {
		key = fmt.Sprintf("%s/%s/backends/%s", basekey, s.Name, s.Backend)
	}

	log.WithFields(log.Fields{
		"key": key,
	}).Debugln("start to delete a service key.")
	if _, err := client.Delete(key, true); err != nil {
		return err
	}
	log.WithFields(log.Fields{
		"name":    s.Name,
		"backend": s.Backend,
		"proto":   s.Proto,
	}).Infoln("service unregistered.")

	return nil
}
Example #4
0
func (etcd *ETCDClusterRunner) deleteDir(client *etcdclient.Client, dir string) {
	responses, err := client.Get(dir)
	Ω(err).ShouldNot(HaveOccured())
	for _, response := range responses {
		if response.Key != "/_etcd" {
			if response.Dir == true {
				etcd.deleteDir(client, response.Key)
			} else {
				_, err := client.Delete(response.Key)
				Ω(err).ShouldNot(HaveOccured())
			}
		}
	}
}
Example #5
0
func TryOccupyTask(client *etcd.Client, name string, taskID uint64, connection string) (bool, error) {
	_, err := client.Create(TaskHealthyPath(name, taskID), "health", 3)
	if err != nil {
		if strings.Contains(err.Error(), "Key already exists") {
			return false, nil
		}
		return false, err
	}
	idStr := strconv.FormatUint(taskID, 10)
	client.Delete(FreeTaskPath(name, idStr), false)
	_, err = client.Set(TaskMasterPath(name, taskID), connection, 0)
	if err != nil {
		return false, err
	}
	return true, nil
}
Example #6
0
func (ec *EtcdCoordinator) parseCommand(c *etcd.Client, resp *etcd.Response) metafora.Command {
	if strings.HasSuffix(resp.Node.Key, MetadataKey) {
		// Skip metadata marker
		return nil
	}

	const recurse = false
	if _, err := c.Delete(resp.Node.Key, recurse); err != nil {
		metafora.Errorf("Error deleting handled command %s: %v", resp.Node.Key, err)
	}

	cmd, err := metafora.UnmarshalCommand([]byte(resp.Node.Value))
	if err != nil {
		metafora.Errorf("Invalid command %s: %v", resp.Node.Key, err)
		return nil
	}
	return cmd
}
Example #7
0
func (locker *EtcdLocker) acquire(client *etcd.Client, key string, ttl uint64, wait bool) (Lock, error) {
	hasLock := false
	key = addPrefix(key)
	lock, err := addLockDirChild(client, key)
	if err != nil {
		return nil, errgo.Mask(err)
	}

	for !hasLock {
		res, err := client.Get(key, true, true)
		if err != nil {
			return nil, errgo.Mask(err)
		}

		if len(res.Node.Nodes) > 1 {
			sort.Sort(res.Node.Nodes)
			if res.Node.Nodes[0].CreatedIndex != lock.Node.CreatedIndex {
				if !wait {
					client.Delete(lock.Node.Key, false)
					return nil, &Error{res.Node.Nodes[0].Value}
				} else {
					err = locker.Wait(lock.Node.Key)
					if err != nil {
						return nil, errgo.Mask(err)
					}
				}
			} else {
				// if the first index is the current one, it's our turn to lock the key
				hasLock = true
			}
		} else {
			// If there are only 1 node, it's our, lock is acquired
			hasLock = true
		}
	}

	// If we get the lock, set the ttl and return it
	_, err = client.Update(lock.Node.Key, lock.Node.Value, ttl)
	if err != nil {
		return nil, errgo.Mask(err)
	}

	return &EtcdLock{client, lock.Node.Key, lock.Node.CreatedIndex}, nil
}
Example #8
0
func etcd_GetDeadKeyDelete() {
	info := config.GetServerInfo()
	log.Printf("Dead info : %s", info.AccessString)

	//key := "SP/Dead"
	key := fmt.Sprintf("%s/Dead", info.AppName)
	log.Printf("Dead list info key : %s\n", key)

	var client *etcd.Client = nil
	client = etcd.NewClient(info.AccessString)

	resp, err := client.Delete(key, true)

	if err != nil {
		log.Fatal(err)
	}

	log.Printf(">>>Dead Key information>>>%s %s", resp.Node.Key, resp.Node.Value)
}
Example #9
0
func (s *SchedulerServer) fetchFrameworkID(client *etcd.Client) (*mesos.FrameworkID, error) {
	if s.failoverTimeout > 0 {
		if response, err := client.Get(meta.FrameworkIDKey, false, false); err != nil {
			if !etcdutil.IsEtcdNotFound(err) {
				return nil, fmt.Errorf("unexpected failure attempting to load framework ID from etcd: %v", err)
			}
			log.V(1).Infof("did not find framework ID in etcd")
		} else if response.Node.Value != "" {
			log.Infof("configuring FrameworkInfo with Id found in etcd: '%s'", response.Node.Value)
			return mutil.NewFrameworkID(response.Node.Value), nil
		}
	} else {
		//TODO(jdef) this seems like a totally hackish way to clean up the framework ID
		if _, err := client.Delete(meta.FrameworkIDKey, true); err != nil {
			if !etcdutil.IsEtcdNotFound(err) {
				return nil, fmt.Errorf("failed to delete framework ID from etcd: %v", err)
			}
			log.V(1).Infof("nothing to delete: did not find framework ID in etcd")
		}
	}
	return nil, nil
}
Example #10
0
func cleanup(c *etcd.Client, key string) {
	if _, err := c.Delete(key, true); err != nil && !isKeyNotFound(err) {
		fmt.Printf("Unable to delete key '%s': %+v", key, err)
	}
}
Example #11
0
func etcdClientTearDown(t *testing.T, rawClient *etcd.Client) {
	if _, err := rawClient.Delete("test", true); err != nil {
		t.Fatal(err)
	}
}
func removeDNS(record string, etcdClient *etcd.Client) error {
	log.Printf("Removing %s from DNS", record)
	_, err := etcdClient.Delete(skymsg.Path(record), true)
	return err
}
func deleteService(client *etcd.Client, service string, addr string) {
	client.Delete(fmt.Sprintf("/services/%s/%s", service, addr))
}