Example #1
0
// resolve all dependencies against configuration
func (self *Resolver) ResolveDependencies(deps []*Dependency) ([]*Library, error) {
	masterLibs := []*Library{}
	resolved := map[string]*Library{}
	results := make(chan *Library)
	errors := make(chan error)
	workQueue := deps

	for len(workQueue) > 0 {
		// de-duplicate the queue
		var err error
		if workQueue, err = self.DeduplicateDeps(workQueue); err != nil {
			return nil, err
		}

		// look for already resolved deps that may match
		if workQueue, err = self.LibResolveDeps(resolved, workQueue); err != nil {
			return nil, err
		}

		// spawn goroutines for each dependency to be resolved
		for _, dep := range workQueue {
			go func(dep *Dependency) {
				lib, err := self.Resolve(dep)
				if err != nil {
					errors <- err
				} else {
					results <- lib
				}
			}(dep)
		}

		// wait on all goroutines to finish or fail
		tempQueue := make([]*Dependency, 0)
		failed := false
		for ii := 0; ii < len(workQueue); ii++ {
			log.Debug("working on %s of  %s", ii, len(workQueue))
			select {
			case lib := <-results:
				log.Debug("Reconciled library: %s", lib.Import)
				resolved[lib.Import] = lib
				masterLibs = append(masterLibs, lib)
				for _, importPath := range lib.Provides {
					log.Debug("Submodule:  %s", importPath)
					resolved[importPath] = lib
				}
				tempQueue = append(tempQueue, lib.Dependencies...)
			case err := <-errors:
				log.Error(err)
				failed = true
			}
		}
		if failed {
			return nil, fmt.Errorf("One or more errors while resolving dependencies.")
		}
		workQueue = tempQueue
	}
	return masterLibs, nil
}
Example #2
0
func installFn(cmd *Command, args []string) error {
	configureLogging()

	if len(args) > 0 {
		return fmt.Errorf("Too many arguments for 'install'")
	}

	// set unset paramters to the defaults
	if lockFileName == "" {
		lockFileName = defaultLockFileName
	}
	if targetPath == "" {
		targetPath = defaultTargetPath
	}

	log.Debug("lock file: %v", lockFileName)
	log.Debug("target path: %v", targetPath)

	// get dependencies from the lockfile
	deplist, err := LoadGrapnelDepsfile(lockFileName)
	if err != nil {
		return err
	} else if deplist == nil {
		// TODO: fail over to update instead?
		return fmt.Errorf("Cannot open lock file: '%s'", lockFileName)
	}
	log.Info("loaded %d dependency definitions", len(deplist))

	log.Info("installing to: %v", targetPath)
	if err := os.MkdirAll(targetPath, 0755); err != nil {
		return err
	}

	libs := []*Library{}
	// cleanup
	defer func() {
		for _, lib := range libs {
			lib.Destroy()
		}
	}()

	// resolve all the dependencies
	resolver, err := getResolver()
	if err != nil {
		return err
	}
	libs, err = resolver.ResolveDependencies(deplist)
	if err != nil {
		return err
	}

	// install all the dependencies
	log.Info("Resolved %v dependencies. Installing.", len(libs))
	resolver.InstallLibraries(targetPath, libs)

	log.Info("Install complete")
	return nil
}
Example #3
0
// apply a match rule
func (self *RewriteRule) Apply(dep *Dependency) error {
	// match *all* expressions against the dependency
	depValues := dep.Flatten()
	for field, match := range self.Matches {
		if !match.MatchString(depValues[field]) {
			return nil // no match
		}
	}

	// generate new value map
	newValues := map[string]string{}
	writer := &bytes.Buffer{}
	for field, tmpl := range self.Replacements {
		writer.Reset()
		if err := tmpl.Execute(writer, depValues); err != nil {
			// TODO: need waaaay more context for this to be useful
			return fmt.Errorf("Error executing replacement rule: %v", err)
		}
		newValues[field] = writer.String()
	}

	// set up the new dependency
	if err := dep.SetValues(newValues); err != nil {
		return err
	}

	log.Debug("Dependency rewritten: %t", dep)

	// return new dependency
	return nil
}
Example #4
0
// Copies a file tree from src to dest
func CopyFileTree(dest string, src string) error {
	return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			log.Info("%s", err.Error())
			return fmt.Errorf("Error while walking file tree")
		}
		relativePath, _ := filepath.Rel(src, path)
		destPath := filepath.Join(dest, relativePath)
		if info.IsDir() {
			// create target directory if it's not already there
			if !Exists(destPath) {
				if err := os.MkdirAll(destPath, 0755); err != nil {
					return err
				}
			}
		} else {
			log.Debug("Copying: %s", destPath)
			if (info.Mode() & os.ModeSymlink) == os.ModeSymlink {
				if err := CopySymlink(path, destPath); err != nil {
					return fmt.Errorf("Could not copy symlink '%s' to '%s'", path, destPath)
				}
			} else if err := CopyFileContents(path, destPath); err != nil {
				return fmt.Errorf("Could not copy file '%s' to '%s'", path, destPath)
			}
		}
		return nil
	})
}
Example #5
0
func (self *RunContext) Run(cmd string, args ...string) error {
	cmdObj := exec.Command(cmd, args...)
	cmdObj.Dir = self.WorkingDirectory
	log.Debug("%v %v", cmd, args)
	out, err := cmdObj.CombinedOutput()
	self.CombinedOutput = string(out)
	if err != nil {
		if _, ok := err.(*exec.ExitError); ok {
			log.Error("%s", out)
		} else {
			log.Error("%s", err.Error())
		}
	}
	return err
}
Example #6
0
func getResolver() (*Resolver, error) {
	resolver := NewResolver()
	resolver.LibSources["git"] = &GitSCM{}
	resolver.LibSources["archive"] = &ArchiveSCM{}

	resolver.AddRewriteRules(BasicRewriteRules)
	resolver.AddRewriteRules(GitRewriteRules)
	resolver.AddRewriteRules(ArchiveRewriteRules)

	// find/validate configuration file
	if configFileName != "" {
		if !Exists(configFileName) {
			return nil, fmt.Errorf("could not locate config file: %s", configFileName)
		}
	} else {
		// search in standard locations
		for _, item := range configFilePath {
			path, err := AbsolutePath(item)
			if err != nil {
				return nil, err
			}
			if Exists(path) {
				configFileName = path
				break
			}
		}
		// warn and exit here if we can't locate on the search path
		if configFileName == "" {
			log.Warn("Could not locate .grapnelrc file; continuing.")
			return resolver, nil
		}
	}

	// load the rules from the config file
	log.Debug("Loading %s", configFileName)
	if rules, err := LoadRewriteRules(configFileName); err != nil {
		return nil, err
	} else {
		resolver.AddRewriteRules(rules)
	}

	return resolver, nil
}
Example #7
0
func updateFn(cmd *Command, args []string) error {
	configureLogging()

	if len(args) > 0 {
		return fmt.Errorf("Too many arguments for 'update'")
	}

	// set unset paramters to the defaults
	if packageFileName == "" {
		packageFileName = defaultPackageFileName
		if lockFileName == "" {
			// set to default iff there was no package filename set
			lockFileName = defaultLockFileName
		}
	} else if lockFileName == "" {
		// compose a new lock file path out of the old package path
		lockFileName = path.Join(path.Dir(packageFileName), "grapnel-lock.toml")
	}
	if targetPath == "" {
		targetPath = defaultTargetPath
	}

	log.Debug("package file: %v", packageFileName)
	log.Debug("lock file: %v", lockFileName)
	log.Debug("target path: %v", targetPath)

	// get dependencies from the grapnel file
	log.Info("loading package file: '%s'", packageFileName)
	deplist, err := LoadGrapnelDepsfile(packageFileName)
	if err != nil {
		return err
	} else if deplist == nil {
		// TODO: fail over to update instead?
		return fmt.Errorf("Cannot open grapnel file: '%s'", packageFileName)
	}
	log.Info("loaded %d dependency definitions", len(deplist))

	// open it now before we expend any real effort
	lockFile, err := os.Create(lockFileName)
	defer lockFile.Close()
	if err != nil {
		log.Error("Cannot open lock file: '%s'", lockFileName)
		return err
	}

	log.Info("installing to: %v", targetPath)
	if err := os.MkdirAll(targetPath, 0755); err != nil {
		return err
	}

	libs := []*Library{}
	// cleanup
	defer func() {
		for _, lib := range libs {
			lib.Destroy()
		}
	}()

	// resolve all the dependencies
	resolver, err := getResolver()
	if err != nil {
		return err
	}
	libs, err = resolver.ResolveDependencies(deplist)
	if err != nil {
		return err
	}

	// install all the dependencies
	log.Info("Resolved %v dependencies. Installing.", len(libs))
	resolver.InstallLibraries(targetPath, libs)

	// write the library data out
	log.Info("Writing lock file")
	for _, lib := range libs {
		lib.ToToml(lockFile)
	}

	if createDsd {
		log.Info("Writing dsd file")
		dsdFileName := path.Join(path.Dir(lockFileName), "grapnel-dsd.sh")
		if err := resolver.ToDsd(dsdFileName, libs); err != nil {
			return err
		}
	}

	log.Info("Update complete")
	return nil
}
Example #8
0
func (self *GitSCM) Resolve(dep *Dependency) (*Library, error) {
	lib := NewLibrary(dep)

	// fix the tag, and default branch
	if lib.Branch == "" {
		lib.Branch = "master"
	}
	if lib.Tag == "" {
		lib.Tag = "HEAD"
	}

	log.Info("Fetching Git Dependency: '%s'", lib.Import)

	// create a dedicated directory and a context for commands
	tempRoot, err := ioutil.TempDir("", "")
	if err != nil {
		return nil, err
	}
	lib.TempDir = tempRoot
	cmd := NewRunContext(tempRoot)

	// use the configured url and acquire the depified branch
	log.Info("Fetching remote data for %s", lib.Import)
	if lib.Url == nil {
		// try all supported protocols against a URL composed from the import
		for _, protocol := range []string{"http", "https", "git", "ssh"} {
			packageUrl := protocol + "://" + lib.Import
			log.Warn("Synthesizing url from import: '%s'", packageUrl)
			if err := cmd.Run("git", "clone", packageUrl, "-b", lib.Branch, tempRoot); err != nil {
				log.Warn("Failed to fetch: '%s'", packageUrl)
				continue
			}
			lib.Url, _ = url.Parse(packageUrl) // pin URL
			break
		}
		if err != nil {
			return nil, fmt.Errorf("Cannot download dependency: '%s'", lib.Import)
		}
	} else if err := cmd.Run("git", "clone", lib.Url.String(), "-b", lib.Branch, tempRoot); err != nil {
		return nil, fmt.Errorf("Cannot download dependency: '%s'", lib.Url.String())
	}

	// move to the specified commit/tag/hash
	// check out a depific commit - may be a tag, commit hash or HEAD
	if err := cmd.Run("git", "checkout", lib.Tag); err != nil {
		return nil, fmt.Errorf("Failed to checkout tag: '%s'", lib.Tag)
	}

	// Pin the Tag to a commit hash if we just have "HEAD" as the 'Tag'
	if lib.Tag == "HEAD" {
		if err := cmd.Run("git", "rev-list", "--all", "--max-count=1"); err != nil {
			return nil, fmt.Errorf("Failed to checkout tag: '%s'", lib.Tag)
		} else {
			lib.Tag = strings.TrimSpace(cmd.CombinedOutput)
		}
	}

	// Stop now if we have no semantic version information
	if lib.VersionSpec.IsUnversioned() {
		lib.Version = NewVersion(-1, -1, -1)
		log.Warn("Resolved: %v (unversioned)", lib.Import)
		stripGitRepo(lib.TempDir)
		return lib, nil
	}

	// find latest version match
	if err := cmd.Run("git", "for-each-ref", "refs/tags", "--sort=taggerdate",
		"--format=%(refname:short)"); err != nil {
		return nil, fmt.Errorf("Failed to acquire ref list for depenency")
	} else {
		for _, line := range strings.Split(cmd.CombinedOutput, "\n") {
			log.Debug("%v", line)
			if ver, err := ParseVersion(line); err == nil {
				log.Debug("ver: %v", ver)
				if dep.VersionSpec.IsSatisfiedBy(ver) {
					lib.Tag = line
					lib.Version = ver
					// move to this tag in the history
					if err := cmd.Run("git", "checkout", lib.Tag); err != nil {
						return nil, fmt.Errorf("Failed to checkout tag: '%s'", lib.Tag)
					}
					break
				}
			} else {
				log.Debug("Parse git tag err: %v", err)
			}
		}
	}

	// fail if the tag cannot be determined.
	if lib.Version == nil {
		return nil, fmt.Errorf("Cannot find a tag for dependency version specification: %v.", lib.VersionSpec)
	}

	log.Info("Resolved: %s %v", lib.Import, lib.Version)
	stripGitRepo(lib.TempDir)
	return lib, nil
}
Example #9
0
func (self *ArchiveSCM) Resolve(dep *Dependency) (*Library, error) {
	lib := NewLibrary(dep)

	// create a dedicated directory and a context for commands
	tempRoot, err := ioutil.TempDir("", "")
	if err != nil {
		return nil, err
	}
	lib.TempDir = tempRoot
	cmd := NewRunContext(tempRoot)

	// prep archive file for write
	filename := filepath.Join(tempRoot, filepath.Base(lib.Dependency.Url.Path))
	file, err := os.OpenFile(filename, os.O_CREATE, 0)
	if err != nil {
		return nil, fmt.Errorf("Cannot open archive for writing: %v", err)
	}
	defer file.Close()

	// get the targeted archive
	response, err := http.Get(lib.Dependency.Url.String())
	if err != nil {
		return nil, fmt.Errorf("Cannot download archive: %v", err)
	}
	if err := response.Write(file); err != nil {
		return nil, fmt.Errorf("Cannot write archive: %v", err)
	}
	file.Close()
	log.Info("Wrote: %s", filename)

	// extract the file
	// TODO: change to using built-in libraries whenever possible.
	switch filepath.Ext(filename) {
	case "zip":
		cmd.Run("unzip", filename)
	case "tar.gz":
		cmd.Run("tar", "xzf", filename)
	case "tar":
		cmd.Run("tar", "xf", filename)
	}
	os.Remove(filename)

	// Stop now if we have no semantic version information
	if lib.VersionSpec.IsUnversioned() {
		lib.Version = NewVersion(-1, -1, -1)
		log.Warn("Resolved: %v (unversioned)", lib.Import)
		return lib, nil
	}

	// get the version number from the filename
	if ver, err := ParseVersion(filepath.Base(filename)); err == nil {
		log.Debug("ver: %v", ver)
		if dep.VersionSpec.IsSatisfiedBy(ver) {
			lib.Version = ver
		}
	} else {
		log.Debug("Parse archive version err: %v", err)
	}

	// fail if the tag cannot be determined.
	if lib.Version == nil {
		return nil, fmt.Errorf("Cannot find a version specification on archive: %v.", lib.VersionSpec)
	}

	log.Info("Resolved: %s %v", lib.Import, lib.Version)
	return lib, nil
}