// resolve all dependencies against configuration
func (self *Resolver) ResolveDependencies(deps []*Dependency) ([]*Library, error) {
masterLibs := []*Library{}
resolved := map[string]*Library{}
results := make(chan *Library)
errors := make(chan error)
workQueue := deps
for len(workQueue) > 0 {
// de-duplicate the queue
var err error
if workQueue, err = self.DeduplicateDeps(workQueue); err != nil {
return nil, err
}
// look for already resolved deps that may match
if workQueue, err = self.LibResolveDeps(resolved, workQueue); err != nil {
return nil, err
}
// spawn goroutines for each dependency to be resolved
for _, dep := range workQueue {
go func(dep *Dependency) {
lib, err := self.Resolve(dep)
if err != nil {
errors <- err
} else {
results <- lib
}
}(dep)
}
// wait on all goroutines to finish or fail
tempQueue := make([]*Dependency, 0)
failed := false
for ii := 0; ii < len(workQueue); ii++ {
log.Debug("working on %s of %s", ii, len(workQueue))
select {
case lib := <-results:
log.Debug("Reconciled library: %s", lib.Import)
resolved[lib.Import] = lib
masterLibs = append(masterLibs, lib)
for _, importPath := range lib.Provides {
log.Debug("Submodule: %s", importPath)
resolved[importPath] = lib
}
tempQueue = append(tempQueue, lib.Dependencies...)
case err := <-errors:
log.Error(err)
failed = true
}
}
if failed {
return nil, fmt.Errorf("One or more errors while resolving dependencies.")
}
workQueue = tempQueue
}
return masterLibs, nil
}
func installFn(cmd *Command, args []string) error {
configureLogging()
if len(args) > 0 {
return fmt.Errorf("Too many arguments for 'install'")
}
// set unset paramters to the defaults
if lockFileName == "" {
lockFileName = defaultLockFileName
}
if targetPath == "" {
targetPath = defaultTargetPath
}
log.Debug("lock file: %v", lockFileName)
log.Debug("target path: %v", targetPath)
// get dependencies from the lockfile
deplist, err := LoadGrapnelDepsfile(lockFileName)
if err != nil {
return err
} else if deplist == nil {
// TODO: fail over to update instead?
return fmt.Errorf("Cannot open lock file: '%s'", lockFileName)
}
log.Info("loaded %d dependency definitions", len(deplist))
log.Info("installing to: %v", targetPath)
if err := os.MkdirAll(targetPath, 0755); err != nil {
return err
}
libs := []*Library{}
// cleanup
defer func() {
for _, lib := range libs {
lib.Destroy()
}
}()
// resolve all the dependencies
resolver, err := getResolver()
if err != nil {
return err
}
libs, err = resolver.ResolveDependencies(deplist)
if err != nil {
return err
}
// install all the dependencies
log.Info("Resolved %v dependencies. Installing.", len(libs))
resolver.InstallLibraries(targetPath, libs)
log.Info("Install complete")
return nil
}
// apply a match rule
func (self *RewriteRule) Apply(dep *Dependency) error {
// match *all* expressions against the dependency
depValues := dep.Flatten()
for field, match := range self.Matches {
if !match.MatchString(depValues[field]) {
return nil // no match
}
}
// generate new value map
newValues := map[string]string{}
writer := &bytes.Buffer{}
for field, tmpl := range self.Replacements {
writer.Reset()
if err := tmpl.Execute(writer, depValues); err != nil {
// TODO: need waaaay more context for this to be useful
return fmt.Errorf("Error executing replacement rule: %v", err)
}
newValues[field] = writer.String()
}
// set up the new dependency
if err := dep.SetValues(newValues); err != nil {
return err
}
log.Debug("Dependency rewritten: %t", dep)
// return new dependency
return nil
}
// Copies a file tree from src to dest
func CopyFileTree(dest string, src string) error {
return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
if err != nil {
log.Info("%s", err.Error())
return fmt.Errorf("Error while walking file tree")
}
relativePath, _ := filepath.Rel(src, path)
destPath := filepath.Join(dest, relativePath)
if info.IsDir() {
// create target directory if it's not already there
if !Exists(destPath) {
if err := os.MkdirAll(destPath, 0755); err != nil {
return err
}
}
} else {
log.Debug("Copying: %s", destPath)
if (info.Mode() & os.ModeSymlink) == os.ModeSymlink {
if err := CopySymlink(path, destPath); err != nil {
return fmt.Errorf("Could not copy symlink '%s' to '%s'", path, destPath)
}
} else if err := CopyFileContents(path, destPath); err != nil {
return fmt.Errorf("Could not copy file '%s' to '%s'", path, destPath)
}
}
return nil
})
}
func (self *RunContext) Run(cmd string, args ...string) error {
cmdObj := exec.Command(cmd, args...)
cmdObj.Dir = self.WorkingDirectory
log.Debug("%v %v", cmd, args)
out, err := cmdObj.CombinedOutput()
self.CombinedOutput = string(out)
if err != nil {
if _, ok := err.(*exec.ExitError); ok {
log.Error("%s", out)
} else {
log.Error("%s", err.Error())
}
}
return err
}
func getResolver() (*Resolver, error) {
resolver := NewResolver()
resolver.LibSources["git"] = &GitSCM{}
resolver.LibSources["archive"] = &ArchiveSCM{}
resolver.AddRewriteRules(BasicRewriteRules)
resolver.AddRewriteRules(GitRewriteRules)
resolver.AddRewriteRules(ArchiveRewriteRules)
// find/validate configuration file
if configFileName != "" {
if !Exists(configFileName) {
return nil, fmt.Errorf("could not locate config file: %s", configFileName)
}
} else {
// search in standard locations
for _, item := range configFilePath {
path, err := AbsolutePath(item)
if err != nil {
return nil, err
}
if Exists(path) {
configFileName = path
break
}
}
// warn and exit here if we can't locate on the search path
if configFileName == "" {
log.Warn("Could not locate .grapnelrc file; continuing.")
return resolver, nil
}
}
// load the rules from the config file
log.Debug("Loading %s", configFileName)
if rules, err := LoadRewriteRules(configFileName); err != nil {
return nil, err
} else {
resolver.AddRewriteRules(rules)
}
return resolver, nil
}
func updateFn(cmd *Command, args []string) error {
configureLogging()
if len(args) > 0 {
return fmt.Errorf("Too many arguments for 'update'")
}
// set unset paramters to the defaults
if packageFileName == "" {
packageFileName = defaultPackageFileName
if lockFileName == "" {
// set to default iff there was no package filename set
lockFileName = defaultLockFileName
}
} else if lockFileName == "" {
// compose a new lock file path out of the old package path
lockFileName = path.Join(path.Dir(packageFileName), "grapnel-lock.toml")
}
if targetPath == "" {
targetPath = defaultTargetPath
}
log.Debug("package file: %v", packageFileName)
log.Debug("lock file: %v", lockFileName)
log.Debug("target path: %v", targetPath)
// get dependencies from the grapnel file
log.Info("loading package file: '%s'", packageFileName)
deplist, err := LoadGrapnelDepsfile(packageFileName)
if err != nil {
return err
} else if deplist == nil {
// TODO: fail over to update instead?
return fmt.Errorf("Cannot open grapnel file: '%s'", packageFileName)
}
log.Info("loaded %d dependency definitions", len(deplist))
// open it now before we expend any real effort
lockFile, err := os.Create(lockFileName)
defer lockFile.Close()
if err != nil {
log.Error("Cannot open lock file: '%s'", lockFileName)
return err
}
log.Info("installing to: %v", targetPath)
if err := os.MkdirAll(targetPath, 0755); err != nil {
return err
}
libs := []*Library{}
// cleanup
defer func() {
for _, lib := range libs {
lib.Destroy()
}
}()
// resolve all the dependencies
resolver, err := getResolver()
if err != nil {
return err
}
libs, err = resolver.ResolveDependencies(deplist)
if err != nil {
return err
}
// install all the dependencies
log.Info("Resolved %v dependencies. Installing.", len(libs))
resolver.InstallLibraries(targetPath, libs)
// write the library data out
log.Info("Writing lock file")
for _, lib := range libs {
lib.ToToml(lockFile)
}
if createDsd {
log.Info("Writing dsd file")
dsdFileName := path.Join(path.Dir(lockFileName), "grapnel-dsd.sh")
if err := resolver.ToDsd(dsdFileName, libs); err != nil {
return err
}
}
log.Info("Update complete")
return nil
}
func (self *GitSCM) Resolve(dep *Dependency) (*Library, error) {
lib := NewLibrary(dep)
// fix the tag, and default branch
if lib.Branch == "" {
lib.Branch = "master"
}
if lib.Tag == "" {
lib.Tag = "HEAD"
}
log.Info("Fetching Git Dependency: '%s'", lib.Import)
// create a dedicated directory and a context for commands
tempRoot, err := ioutil.TempDir("", "")
if err != nil {
return nil, err
}
lib.TempDir = tempRoot
cmd := NewRunContext(tempRoot)
// use the configured url and acquire the depified branch
log.Info("Fetching remote data for %s", lib.Import)
if lib.Url == nil {
// try all supported protocols against a URL composed from the import
for _, protocol := range []string{"http", "https", "git", "ssh"} {
packageUrl := protocol + "://" + lib.Import
log.Warn("Synthesizing url from import: '%s'", packageUrl)
if err := cmd.Run("git", "clone", packageUrl, "-b", lib.Branch, tempRoot); err != nil {
log.Warn("Failed to fetch: '%s'", packageUrl)
continue
}
lib.Url, _ = url.Parse(packageUrl) // pin URL
break
}
if err != nil {
return nil, fmt.Errorf("Cannot download dependency: '%s'", lib.Import)
}
} else if err := cmd.Run("git", "clone", lib.Url.String(), "-b", lib.Branch, tempRoot); err != nil {
return nil, fmt.Errorf("Cannot download dependency: '%s'", lib.Url.String())
}
// move to the specified commit/tag/hash
// check out a depific commit - may be a tag, commit hash or HEAD
if err := cmd.Run("git", "checkout", lib.Tag); err != nil {
return nil, fmt.Errorf("Failed to checkout tag: '%s'", lib.Tag)
}
// Pin the Tag to a commit hash if we just have "HEAD" as the 'Tag'
if lib.Tag == "HEAD" {
if err := cmd.Run("git", "rev-list", "--all", "--max-count=1"); err != nil {
return nil, fmt.Errorf("Failed to checkout tag: '%s'", lib.Tag)
} else {
lib.Tag = strings.TrimSpace(cmd.CombinedOutput)
}
}
// Stop now if we have no semantic version information
if lib.VersionSpec.IsUnversioned() {
lib.Version = NewVersion(-1, -1, -1)
log.Warn("Resolved: %v (unversioned)", lib.Import)
stripGitRepo(lib.TempDir)
return lib, nil
}
// find latest version match
if err := cmd.Run("git", "for-each-ref", "refs/tags", "--sort=taggerdate",
"--format=%(refname:short)"); err != nil {
return nil, fmt.Errorf("Failed to acquire ref list for depenency")
} else {
for _, line := range strings.Split(cmd.CombinedOutput, "\n") {
log.Debug("%v", line)
if ver, err := ParseVersion(line); err == nil {
log.Debug("ver: %v", ver)
if dep.VersionSpec.IsSatisfiedBy(ver) {
lib.Tag = line
lib.Version = ver
// move to this tag in the history
if err := cmd.Run("git", "checkout", lib.Tag); err != nil {
return nil, fmt.Errorf("Failed to checkout tag: '%s'", lib.Tag)
}
break
}
} else {
log.Debug("Parse git tag err: %v", err)
}
}
}
// fail if the tag cannot be determined.
if lib.Version == nil {
return nil, fmt.Errorf("Cannot find a tag for dependency version specification: %v.", lib.VersionSpec)
}
log.Info("Resolved: %s %v", lib.Import, lib.Version)
stripGitRepo(lib.TempDir)
return lib, nil
}
func (self *ArchiveSCM) Resolve(dep *Dependency) (*Library, error) {
lib := NewLibrary(dep)
// create a dedicated directory and a context for commands
tempRoot, err := ioutil.TempDir("", "")
if err != nil {
return nil, err
}
lib.TempDir = tempRoot
cmd := NewRunContext(tempRoot)
// prep archive file for write
filename := filepath.Join(tempRoot, filepath.Base(lib.Dependency.Url.Path))
file, err := os.OpenFile(filename, os.O_CREATE, 0)
if err != nil {
return nil, fmt.Errorf("Cannot open archive for writing: %v", err)
}
defer file.Close()
// get the targeted archive
response, err := http.Get(lib.Dependency.Url.String())
if err != nil {
return nil, fmt.Errorf("Cannot download archive: %v", err)
}
if err := response.Write(file); err != nil {
return nil, fmt.Errorf("Cannot write archive: %v", err)
}
file.Close()
log.Info("Wrote: %s", filename)
// extract the file
// TODO: change to using built-in libraries whenever possible.
switch filepath.Ext(filename) {
case "zip":
cmd.Run("unzip", filename)
case "tar.gz":
cmd.Run("tar", "xzf", filename)
case "tar":
cmd.Run("tar", "xf", filename)
}
os.Remove(filename)
// Stop now if we have no semantic version information
if lib.VersionSpec.IsUnversioned() {
lib.Version = NewVersion(-1, -1, -1)
log.Warn("Resolved: %v (unversioned)", lib.Import)
return lib, nil
}
// get the version number from the filename
if ver, err := ParseVersion(filepath.Base(filename)); err == nil {
log.Debug("ver: %v", ver)
if dep.VersionSpec.IsSatisfiedBy(ver) {
lib.Version = ver
}
} else {
log.Debug("Parse archive version err: %v", err)
}
// fail if the tag cannot be determined.
if lib.Version == nil {
return nil, fmt.Errorf("Cannot find a version specification on archive: %v.", lib.VersionSpec)
}
log.Info("Resolved: %s %v", lib.Import, lib.Version)
return lib, nil
}