func (t *Tree) buildProviderAliasGraph(g *dag.AcyclicGraph, parent dag.Vertex) {
// Add all our defined aliases
defined := make(map[string]struct{})
for _, p := range t.config.ProviderConfigs {
defined[p.FullName()] = struct{}{}
}
// Add all our used aliases
used := make(map[string]struct{})
for _, r := range t.config.Resources {
if r.Provider != "" {
used[r.Provider] = struct{}{}
}
}
// Add it to the graph
vertex := &providerAliasVertex{
Path: t.Path(),
Defined: defined,
Used: used,
}
g.Add(vertex)
// Connect to our parent if we have one
if parent != nil {
g.Connect(dag.BasicEdge(vertex, parent))
}
// Build all our children
for _, c := range t.Children() {
c.buildProviderAliasGraph(g, vertex)
}
}
func (l *Layered) graph(db *bolt.DB) (*dag.AcyclicGraph, error) {
graph := new(dag.AcyclicGraph)
graph.Add("root")
// First, add all the layers
layers := make(map[string]*layerVertex)
err := db.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket(boltLayersBucket)
return bucket.ForEach(func(k, data []byte) error {
var v layerVertex
if err := l.structRead(&v, data); err != nil {
return err
}
// Add this layer to the graph
graph.Add(&v)
// Store the mapping for later
layers[v.Layer.ID] = &v
return nil
})
})
if err != nil {
return nil, err
}
// Next, connect the layers
err = db.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket(boltEdgesBucket)
return bucket.ForEach(func(k, data []byte) error {
from := layers[string(k)]
to := layers[string(data)]
if from != nil && to != nil {
graph.Connect(dag.BasicEdge(from, to))
}
return nil
})
})
if err != nil {
return nil, err
}
// Finally, add and connect all the envs
err = db.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket(boltEnvsBucket)
return bucket.ForEach(func(k, data []byte) error {
key := fmt.Sprintf("env-%s", string(k))
graph.Add(key)
// Connect the env to the layer it depends on
to := &layerVertex{Layer: &Layer{ID: string(data)}}
graph.Connect(dag.BasicEdge(key, to))
// Connect the root to the environment that is active
graph.Connect(dag.BasicEdge("root", key))
return nil
})
})
if err != nil {
return nil, err
}
return graph, nil
}
// compileImports takes a File, loads all the imports, and merges them
// into the File.
func (c *Compiler) compileImports(root *File) error {
// If we have no imports, short-circuit the whole thing
if len(root.Imports) == 0 {
return nil
}
// Pull these out into variables so they're easier to reference
storage := c.importStorage
cache := c.importCache
cacheLock := &c.importLock
// A graph is used to track for cycles
var graphLock sync.Mutex
graph := new(dag.AcyclicGraph)
graph.Add("root")
// Since we run the import in parallel, multiple errors can happen
// at the same time. We use multierror and a lock to keep track of errors.
var resultErr error
var resultErrLock sync.Mutex
// Forward declarations for some nested functions we use. The docs
// for these functions are above each.
var importSingle func(parent string, f *File) bool
var downloadSingle func(string, *sync.WaitGroup, *sync.Mutex, []*File, int)
// importSingle is responsible for kicking off the imports and merging
// them for a single file. This will return true on success, false on
// failure. On failure, it is expected that any errors are appended to
// resultErr.
importSingle = func(parent string, f *File) bool {
var wg sync.WaitGroup
// Build the list of files we'll merge later
var mergeLock sync.Mutex
merge := make([]*File, len(f.Imports))
// Go through the imports and kick off the download
for idx, i := range f.Imports {
source, err := getter.Detect(
i.Source, filepath.Dir(f.Path),
getter.Detectors)
if err != nil {
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"Error loading import source: %s", err))
return false
}
// Add this to the graph and check now if there are cycles
graphLock.Lock()
graph.Add(source)
graph.Connect(dag.BasicEdge(parent, source))
cycles := graph.Cycles()
graphLock.Unlock()
if len(cycles) > 0 {
for _, cycle := range cycles {
names := make([]string, len(cycle))
for i, v := range cycle {
names[i] = dag.VertexName(v)
}
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"Cycle found: %s", strings.Join(names, ", ")))
return false
}
}
wg.Add(1)
go downloadSingle(source, &wg, &mergeLock, merge, idx)
}
// Wait for completion
wg.Wait()
// Go through the merge list and look for any nil entries, which
// means that download failed. In that case, return immediately.
// We assume any errors were put into resultErr.
for _, importF := range merge {
if importF == nil {
return false
}
}
for _, importF := range merge {
// We need to copy importF here so that we don't poison
// the cache by modifying the same pointer.
importFCopy := *importF
importF = &importFCopy
source := importF.ID
importF.ID = ""
importF.Path = ""
// Merge it into our file!
if err := f.Merge(importF); err != nil {
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"Error merging import %s: %s", source, err))
return false
}
}
return true
}
// downloadSingle is used to download a single import and parse the
// Appfile. This is a separate function because it is generally run
// in a goroutine so we can parallelize grabbing the imports.
downloadSingle = func(source string, wg *sync.WaitGroup, l *sync.Mutex, result []*File, idx int) {
defer wg.Done()
// Read from the cache if we have it
cacheLock.Lock()
cached, ok := cache[source]
cacheLock.Unlock()
if ok {
log.Printf("[DEBUG] cache hit on import: %s", source)
l.Lock()
defer l.Unlock()
result[idx] = cached
return
}
// Call the callback if we have one
log.Printf("[DEBUG] loading import: %s", source)
if c.opts.Callback != nil {
c.opts.Callback(&CompileEventImport{
Source: source,
})
}
// Download the dependency
if err := storage.Get(source, source, true); err != nil {
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"Error loading import source: %s", err))
return
}
dir, _, err := storage.Dir(source)
if err != nil {
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"Error loading import source: %s", err))
return
}
// Parse the Appfile
importF, err := ParseFile(filepath.Join(dir, "Appfile"))
if err != nil {
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"Error parsing Appfile in %s: %s", source, err))
return
}
// We use the ID to store the source, but we clear it
// when we actually merge.
importF.ID = source
// Import the imports in this
if !importSingle(source, importF) {
return
}
// Once we're done, acquire the lock and write it
l.Lock()
result[idx] = importF
l.Unlock()
// Write this into the cache.
cacheLock.Lock()
cache[source] = importF
cacheLock.Unlock()
}
importSingle("root", root)
return resultErr
}
func (c *Compiler) compileDependencies(root *CompiledGraphVertex, graph *dag.AcyclicGraph) error {
// For easier reference below
storage := c.depStorage
// Make a map to keep track of the dep source to vertex mapping
vertexMap := make(map[string]*CompiledGraphVertex)
// Store ourselves in the map
key, err := getter.Detect(
".", filepath.Dir(root.File.Path),
getter.Detectors)
if err != nil {
return err
}
vertexMap[key] = root
// Make a queue for the other vertices we need to still get
// dependencies for. We arbitrarily make the cap for this slice
// 30, since that is a ton of dependencies and we don't expect the
// average case to have more than this.
queue := make([]*CompiledGraphVertex, 1, 30)
queue[0] = root
// While we still have dependencies to get, continue loading them.
// TODO: parallelize
for len(queue) > 0 {
var current *CompiledGraphVertex
current, queue = queue[len(queue)-1], queue[:len(queue)-1]
log.Printf("[DEBUG] compiling dependencies for: %s", current.Name())
for _, dep := range current.File.Application.Dependencies {
key, err := getter.Detect(
dep.Source, filepath.Dir(current.File.Path),
getter.Detectors)
if err != nil {
return fmt.Errorf(
"Error loading source: %s", err)
}
vertex := vertexMap[key]
if vertex == nil {
log.Printf("[DEBUG] loading dependency: %s", key)
// Call the callback if we have one
if c.opts.Callback != nil {
c.opts.Callback(&CompileEventDep{
Source: key,
})
}
// Download the dependency
if err := storage.Get(key, key, true); err != nil {
return err
}
dir, _, err := storage.Dir(key)
if err != nil {
return err
}
// Parse the Appfile if it exists
var f *File
appfilePath := filepath.Join(dir, "Appfile")
_, err = os.Stat(appfilePath)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf(
"Error parsing Appfile in %s: %s", key, err)
}
if err == nil {
f, err = ParseFile(appfilePath)
if err != nil {
return fmt.Errorf(
"Error parsing Appfile in %s: %s", key, err)
}
// Realize all the imports for this file
if err := c.compileImports(f); err != nil {
return err
}
}
// Do any additional loading if we have a loader
if c.opts.Loader != nil {
f, err = c.opts.Loader(f, dir)
if err != nil {
return fmt.Errorf(
"Error loading Appfile in %s: %s", key, err)
}
}
// Set the source
f.Source = key
// If it doesn't have an otto ID then we can't do anything
hasID, err := f.hasID()
if err != nil {
return fmt.Errorf(
"Error checking for ID file for Appfile in %s: %s",
key, err)
}
if !hasID {
return fmt.Errorf(
"Dependency '%s' doesn't have an Otto ID yet!\n\n"+
"An Otto ID is generated on the first compilation of the Appfile.\n"+
"It is a globally unique ID that is used to track the application\n"+
"across multiple deploys. It is required for the application to be\n"+
"used as a dependency. To fix this, check out that application and\n"+
"compile the Appfile with `otto compile` once. Make sure you commit\n"+
"the .ottoid file into version control, and then try this command\n"+
"again.",
key)
}
// We merge the root infrastructure choice upwards to
// all dependencies.
f.Infrastructure = root.File.Infrastructure
if root.File.Project != nil {
if f.Project == nil {
f.Project = new(Project)
}
f.Project.Infrastructure = root.File.Project.Infrastructure
}
// Build the vertex for this
vertex = &CompiledGraphVertex{
File: f,
Dir: dir,
NameValue: f.Application.Name,
}
// Add the vertex since it is new, store the mapping, and
// queue it to be loaded later.
graph.Add(vertex)
vertexMap[key] = vertex
queue = append(queue, vertex)
}
// Connect the dependencies
graph.Connect(dag.BasicEdge(current, vertex))
}
}
return nil
}
// compileImports需要一个文件，load所有的imports,然后合并到文件中
func compileImports(root *File, importOpts *compileImportOpts, opts *CompileOpts) error {
// 如果没有imports，直接短路(返回)
if len(root.Imports) == 0 {
return nil
}
// 把它们放入变量，以便我们可以更早的引用
//storage := importOpts.Storage
//cache := importOpts.Cache
//cacheLock := importOpts.CacheLock
// A graph is used to track for cycles
var graphLock sync.Mutex
graph := new(dag.AcyclicGraph)
graph.Add("root")
// 自从我们平行的执行导入，在同一时间会有多个error发生
// 我们使用multierror lock和跟踪error
var resultErr error
var resultErrLock sync.Mutex
// Forward declarations for some nested functions we use. The docs
// for these functions are above each.
var importSingle func(parent string, f *File) bool
var downloadSingle func(string, *sync.WaitGroup, *sync.Mutex, []*File, int)
// importSingle is responsible for kicking off the imports and merging
// them for a single file. This will return true on success, false on
// failure. On failure, it is expected that any errors are appended to
// resultErr.
importSingle = func(parent string, f *File) bool {
var wg sync.WaitGroup
// 构建文件列表，后面将合并
var mergeLock sync.Mutex
merge := make([]*File, len(f.Imports))
// 通过导入并开始处理下载
for idx, i := range f.Imports {
source, err := getter.Detect(i.Source, filepath.Dir(f.Path), getter.Detectors)
if err != nil {
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"获取import源错误: %s", err))
return false
}
// Add this to the graph and check now if there are cycles
graphLock.Lock()
graph.Add(source)
graph.Connect(dag.BasicEdge(parent, source))
cycles := graph.Cycles()
graphLock.Unlock()
if len(cycles) > 0 {
for _, cycle := range cycles {
names := make([]string, len(cycle))
for i, v := range cycle {
names[i] = dag.VertexName(v)
}
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"Cycle found: %s", strings.Join(names, ", ")))
return false
}
}
wg.Add(1)
go downloadSingle(source, &wg, &mergeLock, merge, idx)
}
// Wait for completion
wg.Wait()
// Go through the merge list and look for any nil entries, which
// means that download failed. In that case, return immediately.
// We assume any errors were put into resultErr.
for _, importF := range merge {
if importF == nil {
return false
}
}
for _, importF := range merge {
// We need to copy importF here so that we don't poison
// the cache by modifying the same pointer.
importFCopy := *importF
importF = &importFCopy
source := importF.ID
importF.ID = ""
importF.Path = ""
// Merge it into our file!
if err := f.Merge(importF); err != nil {
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"合并import错误 %s : %s", source, err))
return false
}
}
return true
}
// downloadSingle is used to download a single import and parse the
// Appfile. This is a separate function because it is generally run
// in a goroutine so we can parallelize grabbing the imports.
downloadSingle = func(source string, wg *sync.WaitGroup, l *sync.Mutex, result []*File, idx int) {}
importSingle("root", root)
return resultErr
}
// validateProviderAlias validates that all provider alias references are
// defined at some point in the parent tree. This improves UX by catching
// alias typos at the slight cost of requiring a declaration of usage. This
// is usually a good tradeoff since not many aliases are used.
func (t *Tree) validateProviderAlias() error {
// If we're not the root, don't perform this validation. We must be the
// root since we require full tree visibilty.
if len(t.path) != 0 {
return nil
}
// We'll use a graph to keep track of defined aliases at each level.
// As long as a parent defines an alias, it is okay.
var g dag.AcyclicGraph
t.buildProviderAliasGraph(&g, nil)
// Go through the graph and check that the usage is all good.
var err error
for _, v := range g.Vertices() {
pv, ok := v.(*providerAliasVertex)
if !ok {
// This shouldn't happen, just ignore it.
continue
}
// If we're not using any aliases, fast track and just continue
if len(pv.Used) == 0 {
continue
}
// Grab the ancestors since we're going to have to check if our
// parents define any of our aliases.
var parents []*providerAliasVertex
ancestors, _ := g.Ancestors(v)
for _, raw := range ancestors.List() {
if pv, ok := raw.(*providerAliasVertex); ok {
parents = append(parents, pv)
}
}
for k, _ := range pv.Used {
// Check if we define this
if _, ok := pv.Defined[k]; ok {
continue
}
// Check for a parent
found := false
for _, parent := range parents {
_, found = parent.Defined[k]
if found {
break
}
}
if found {
continue
}
// We didn't find the alias, error!
err = multierror.Append(err, fmt.Errorf(
"module %s: provider alias must be defined by the module or a parent: %s",
strings.Join(pv.Path, "."), k))
}
}
return err
}
