func TestPruneNoopTransformer(t *testing.T) {
g := Graph{Path: RootModulePath}
a := &testGraphNodeNoop{NameValue: "A"}
b := &testGraphNodeNoop{NameValue: "B", Value: true}
c := &testGraphNodeNoop{NameValue: "C"}
g.Add(a)
g.Add(b)
g.Add(c)
g.Connect(dag.BasicEdge(a, b))
g.Connect(dag.BasicEdge(b, c))
{
tf := &PruneNoopTransformer{}
if err := tf.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
}
}
actual := strings.TrimSpace(g.String())
expected := strings.TrimSpace(testTransformPruneNoopStr)
if actual != expected {
t.Fatalf("bad:\n\n%s", actual)
}
}
func TestVertexTransformer(t *testing.T) {
var g Graph
g.Add(1)
g.Add(2)
g.Add(3)
g.Connect(dag.BasicEdge(1, 2))
g.Connect(dag.BasicEdge(2, 3))
{
tf := &VertexTransformer{
Transforms: []GraphVertexTransformer{
&testVertexTransform{Source: 2, Target: 42},
},
}
if err := tf.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
}
}
actual := strings.TrimSpace(g.String())
expected := strings.TrimSpace(testVertexTransformerStr)
if actual != expected {
t.Fatalf("bad: %s", actual)
}
}
func (t *CreateBeforeDestroyTransformer) Transform(g *Graph) error {
// We "stage" the edge connections/destroys in these slices so that
// while we're doing the edge transformations (transpositions) in
// the graph, we're not affecting future edge transpositions. These
// slices let us stage ALL the changes that WILL happen so that all
// of the transformations happen atomically.
var connect, destroy []dag.Edge
for _, v := range g.Vertices() {
// We only care to use the destroy nodes
dn, ok := v.(GraphNodeDestroy)
if !ok {
continue
}
// If the node doesn't need to create before destroy, then continue
if !dn.CreateBeforeDestroy() {
continue
}
// Get the creation side of this node
cn := dn.CreateNode()
// Take all the things which depend on the creation node and
// make them dependencies on the destruction. Clarifying this
// with an example: if you have a web server and a load balancer
// and the load balancer depends on the web server, then when we
// do a create before destroy, we want to make sure the steps are:
//
// 1.) Create new web server
// 2.) Update load balancer
// 3.) Delete old web server
//
// This ensures that.
for _, sourceRaw := range g.UpEdges(cn).List() {
source := sourceRaw.(dag.Vertex)
connect = append(connect, dag.BasicEdge(dn, source))
}
// Swap the edge so that the destroy depends on the creation
// happening...
connect = append(connect, dag.BasicEdge(dn, cn))
destroy = append(destroy, dag.BasicEdge(cn, dn))
}
for _, edge := range connect {
g.Connect(edge)
}
for _, edge := range destroy {
g.RemoveEdge(edge)
}
return nil
}
func (t *Tree) buildProviderAliasGraph(g *dag.AcyclicGraph, parent dag.Vertex) {
// Add all our defined aliases
defined := make(map[string]struct{})
for _, p := range t.config.ProviderConfigs {
defined[p.FullName()] = struct{}{}
}
// Add all our used aliases
used := make(map[string]struct{})
for _, r := range t.config.Resources {
if r.Provider != "" {
used[r.Provider] = struct{}{}
}
}
// Add it to the graph
vertex := &providerAliasVertex{
Path: t.Path(),
Defined: defined,
Used: used,
}
g.Add(vertex)
// Connect to our parent if we have one
if parent != nil {
g.Connect(dag.BasicEdge(vertex, parent))
}
// Build all our children
for _, c := range t.Children() {
c.buildProviderAliasGraph(g, vertex)
}
}
// ConnectFrom creates an edge by finding the source from a DependableName
// and connecting it to the specific vertex.
func (g *Graph) ConnectFrom(source string, target dag.Vertex) {
g.once.Do(g.init)
if source := g.dependableMap[source]; source != nil {
g.Connect(dag.BasicEdge(source, target))
}
}
func (t *ProxyTransformer) Transform(g *Graph) error {
for _, v := range g.Vertices() {
pn, ok := v.(GraphNodeProxy)
if !ok {
continue
}
// If we don't want to be proxies, don't do it
if !pn.Proxy() {
continue
}
// Connect all the things that depend on this to things that
// we depend on as the proxy. See docs for GraphNodeProxy for
// a visual explanation.
for _, s := range g.UpEdges(v).List() {
for _, t := range g.DownEdges(v).List() {
g.Connect(GraphProxyEdge{
Edge: dag.BasicEdge(s, t),
})
}
}
}
return nil
}
func (c *Compiled) UnmarshalJSON(data []byte) error {
var raw compiledJSON
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
c.File = raw.File
c.Graph = new(dag.AcyclicGraph)
for _, v := range raw.Vertices {
c.Graph.Add(v)
}
for _, e := range raw.Edges {
for a, b := range e {
ai, err := strconv.ParseInt(a, 0, 0)
if err != nil {
return err
}
bi, err := strconv.ParseInt(b, 0, 0)
if err != nil {
return err
}
c.Graph.Connect(dag.BasicEdge(raw.Vertices[ai], raw.Vertices[bi]))
}
}
return nil
}
func TestExpandTransform(t *testing.T) {
var g Graph
g.Add(1)
g.Add(2)
g.Connect(dag.BasicEdge(1, 2))
tf := &ExpandTransform{}
out, err := tf.Transform(&testExpandable{
Result: &g,
})
if err != nil {
t.Fatalf("err: %s", err)
}
sn, ok := out.(GraphNodeSubgraph)
if !ok {
t.Fatalf("not subgraph: %#v", out)
}
actual := strings.TrimSpace(sn.Subgraph().String())
expected := strings.TrimSpace(testExpandTransformStr)
if actual != expected {
t.Fatalf("bad: %s", actual)
}
}
func (t *PruneNoopTransformer) Transform(g *Graph) error {
// Find the leaves.
leaves := make([]dag.Vertex, 0, 10)
for _, v := range g.Vertices() {
if g.DownEdges(v).Len() == 0 {
leaves = append(leaves, v)
}
}
// Do a depth first walk from the leaves and remove things.
return g.ReverseDepthFirstWalk(leaves, func(v dag.Vertex, depth int) error {
// We need a prunable
pn, ok := v.(GraphNodeNoopPrunable)
if !ok {
return nil
}
// Start building the noop opts
path := g.Path
if pn, ok := v.(GraphNodeSubPath); ok {
path = pn.Path()
}
var modDiff *ModuleDiff
var modState *ModuleState
if t.Diff != nil {
modDiff = t.Diff.ModuleByPath(path)
}
if t.State != nil {
modState = t.State.ModuleByPath(path)
}
// Determine if its a noop. If it isn't, just return
noop := pn.Noop(&NoopOpts{
Graph: g,
Vertex: v,
Diff: t.Diff,
State: t.State,
ModDiff: modDiff,
ModState: modState,
})
if !noop {
return nil
}
// It is a noop! We first preserve edges.
up := g.UpEdges(v).List()
for _, downV := range g.DownEdges(v).List() {
for _, upV := range up {
g.Connect(dag.BasicEdge(upV, downV))
}
}
// Then remove it
g.Remove(v)
return nil
})
}
func TestDebugJSON2Dot(t *testing.T) {
// create the graph JSON output
logFile, err := ioutil.TempFile("", "tf")
if err != nil {
t.Fatal(err)
}
defer os.Remove(logFile.Name())
var g dag.Graph
g.SetDebugWriter(logFile)
g.Add(1)
g.Add(2)
g.Add(3)
g.Connect(dag.BasicEdge(1, 2))
g.Connect(dag.BasicEdge(2, 3))
ui := new(cli.MockUi)
c := &DebugJSON2DotCommand{
Meta: Meta{
ContextOpts: testCtxConfig(testProvider()),
Ui: ui,
},
}
args := []string{
logFile.Name(),
}
if code := c.Run(args); code != 0 {
t.Fatalf("bad: \n%s", ui.ErrorWriter.String())
}
output := ui.OutputWriter.String()
if !strings.HasPrefix(output, "digraph {") {
t.Fatalf("doesn't look like digraph: %s", output)
}
if !strings.Contains(output, `subgraph "root" {`) {
t.Fatalf("doesn't contains root subgraph: %s", output)
}
}
func (t *ParentProviderTransformer) Transform(g *Graph) error {
// Make a mapping of path to dag.Vertex, where path is: "path.name"
m := make(map[string]dag.Vertex)
// Also create a map that maps a provider to its parent
parentMap := make(map[dag.Vertex]string)
for _, raw := range g.Vertices() {
// If it is the flat version, then make it the non-flat version.
// We eventually want to get rid of the flat version entirely so
// this is a stop-gap while it still exists.
var v dag.Vertex = raw
if f, ok := v.(*graphNodeProviderFlat); ok {
v = f.graphNodeProvider
}
// Only care about providers
pn, ok := v.(GraphNodeProvider)
if !ok || pn.ProviderName() == "" {
continue
}
// Also require a subpath, if there is no subpath then we
// just totally ignore it. The expectation of this transform is
// that it is used with a graph builder that is already flattened.
var path []string
if pn, ok := raw.(GraphNodeSubPath); ok {
path = pn.Path()
}
path = normalizeModulePath(path)
// Build the key with path.name i.e. "child.subchild.aws"
key := fmt.Sprintf("%s.%s", strings.Join(path, "."), pn.ProviderName())
m[key] = raw
// Determine the parent if we're non-root. This is length 1 since
// the 0 index should be "root" since we normalize above.
if len(path) > 1 {
path = path[:len(path)-1]
key := fmt.Sprintf("%s.%s", strings.Join(path, "."), pn.ProviderName())
parentMap[raw] = key
}
}
// Connect!
for v, key := range parentMap {
if parent, ok := m[key]; ok {
g.Connect(dag.BasicEdge(v, parent))
}
}
return nil
}
func TestProxyTransformer(t *testing.T) {
var g Graph
proxy := &testNodeProxy{NameValue: "proxy"}
g.Add("A")
g.Add("C")
g.Add(proxy)
g.Connect(dag.BasicEdge("A", proxy))
g.Connect(dag.BasicEdge(proxy, "C"))
{
tf := &ProxyTransformer{}
if err := tf.Transform(&g); err != nil {
t.Fatalf("err: %s", err)
}
}
actual := strings.TrimSpace(g.String())
expected := strings.TrimSpace(testProxyTransformStr)
if actual != expected {
t.Fatalf("bad: %s", actual)
}
}
func (t *CloseProviderTransformer) Transform(g *Graph) error {
pm := providerVertexMap(g)
cpm := closeProviderVertexMap(g)
var err error
for _, v := range g.Vertices() {
if pv, ok := v.(GraphNodeProviderConsumer); ok {
for _, p := range pv.ProvidedBy() {
key := p
source := cpm[key]
if source == nil {
// Create a new graphNodeCloseProvider and add it to the graph
source = &graphNodeCloseProvider{ProviderNameValue: p}
g.Add(source)
// Close node needs to depend on provider
provider, ok := pm[key]
if !ok {
err = multierror.Append(err, fmt.Errorf(
"%s: provider %s couldn't be found for closing",
dag.VertexName(v), p))
continue
}
g.Connect(dag.BasicEdge(source, provider))
// Make sure we also add the new graphNodeCloseProvider to the map
// so we don't create and add any duplicate graphNodeCloseProviders.
cpm[key] = source
}
// Close node depends on all nodes provided by the provider
g.Connect(dag.BasicEdge(source, v))
}
}
}
return err
}
// ConnectTo connects a vertex to a raw string of targets that are the
// result of DependableName, and returns the list of targets that are missing.
func (g *Graph) ConnectTo(v dag.Vertex, targets []string) []string {
g.once.Do(g.init)
var missing []string
for _, t := range targets {
if dest := g.dependableMap[t]; dest != nil {
g.Connect(dag.BasicEdge(v, dest))
} else {
missing = append(missing, t)
}
}
return missing
}
func (t *CountBoundaryTransformer) Transform(g *Graph) error {
node := &NodeCountBoundary{}
g.Add(node)
// Depends on everything
for _, v := range g.Vertices() {
// Don't connect to ourselves
if v == node {
continue
}
// Connect!
g.Connect(dag.BasicEdge(node, v))
}
return nil
}
func (t *ModuleInputTransformer) Transform(g *Graph) error {
// Create the node
n := &graphNodeModuleInput{Variables: t.Variables}
// Add it to the graph
g.Add(n)
// Connect the inputs to the bottom of the graph so that it happens
// first.
for _, v := range g.Vertices() {
if v == n {
continue
}
if g.DownEdges(v).Len() == 0 {
g.Connect(dag.BasicEdge(v, n))
}
}
return nil
}
func (t *ProvisionerTransformer) Transform(g *Graph) error {
// Go through the other nodes and match them to provisioners they need
var err error
m := provisionerVertexMap(g)
for _, v := range g.Vertices() {
if pv, ok := v.(GraphNodeProvisionerConsumer); ok {
for _, p := range pv.ProvisionedBy() {
if m[p] == nil {
err = multierror.Append(err, fmt.Errorf(
"%s: provisioner %s couldn't be found",
dag.VertexName(v), p))
continue
}
g.Connect(dag.BasicEdge(v, m[p]))
}
}
}
return err
}
func (t *RootTransformer) Transform(g *Graph) error {
// If we already have a good root, we're done
if _, err := g.Root(); err == nil {
return nil
}
// Add a root
var root graphNodeRoot
g.Add(root)
// Connect the root to all the edges that need it
for _, v := range g.Vertices() {
if v == root {
continue
}
if g.UpEdges(v).Len() == 0 {
g.Connect(dag.BasicEdge(root, v))
}
}
return nil
}
func (t *ProviderTransformer) Transform(g *Graph) error {
// Go through the other nodes and match them to providers they need
var err error
m := providerVertexMap(g)
for _, v := range g.Vertices() {
if pv, ok := v.(GraphNodeProviderConsumer); ok {
for _, p := range pv.ProvidedBy() {
target := m[providerMapKey(p, pv)]
if target == nil {
println(fmt.Sprintf("%#v\n\n%#v", m, providerMapKey(p, pv)))
err = multierror.Append(err, fmt.Errorf(
"%s: provider %s couldn't be found",
dag.VertexName(v), p))
continue
}
g.Connect(dag.BasicEdge(v, target))
}
}
}
return err
}
func (t *ReferenceTransformer) Transform(g *Graph) error {
// Build a reference map so we can efficiently look up the references
vs := g.Vertices()
m := NewReferenceMap(vs)
// Find the things that reference things and connect them
for _, v := range vs {
parents, _ := m.References(v)
parentsDbg := make([]string, len(parents))
for i, v := range parents {
parentsDbg[i] = dag.VertexName(v)
}
log.Printf(
"[DEBUG] ReferenceTransformer: %q references: %v",
dag.VertexName(v), parentsDbg)
for _, parent := range parents {
g.Connect(dag.BasicEdge(v, parent))
}
}
return nil
}
func (t *CloseProviderTransformer) Transform(g *Graph) error {
m := closeProviderVertexMap(g)
for _, v := range g.Vertices() {
if pv, ok := v.(GraphNodeProviderConsumer); ok {
for _, p := range pv.ProvidedBy() {
source := m[p]
if source == nil {
// Create a new graphNodeCloseProvider and add it to the graph
source = &graphNodeCloseProvider{ProviderNameValue: p}
g.Add(source)
// Make sure we also add the new graphNodeCloseProvider to the map
// so we don't create and add any duplicate graphNodeCloseProviders.
m[p] = source
}
g.Connect(dag.BasicEdge(source, v))
}
}
}
return nil
}
func (t *DestroyEdgeTransformer) Transform(g *Graph) error {
log.Printf("[TRACE] DestroyEdgeTransformer: Beginning destroy edge transformation...")
// Build a map of what is being destroyed (by address string) to
// the list of destroyers. In general there will only be one destroyer
// but to make it more robust we support multiple.
destroyers := make(map[string][]GraphNodeDestroyer)
for _, v := range g.Vertices() {
dn, ok := v.(GraphNodeDestroyer)
if !ok {
continue
}
addr := dn.DestroyAddr()
if addr == nil {
continue
}
key := addr.String()
log.Printf(
"[TRACE] DestroyEdgeTransformer: %s destroying %q",
dag.VertexName(dn), key)
destroyers[key] = append(destroyers[key], dn)
}
// If we aren't destroying anything, there will be no edges to make
// so just exit early and avoid future work.
if len(destroyers) == 0 {
return nil
}
// Go through and connect creators to destroyers. Going along with
// our example, this makes: A_d => A
for _, v := range g.Vertices() {
cn, ok := v.(GraphNodeCreator)
if !ok {
continue
}
addr := cn.CreateAddr()
if addr == nil {
continue
}
key := addr.String()
ds := destroyers[key]
if len(ds) == 0 {
continue
}
for _, d := range ds {
// For illustrating our example
a_d := d.(dag.Vertex)
a := v
log.Printf(
"[TRACE] DestroyEdgeTransformer: connecting creator/destroyer: %s, %s",
dag.VertexName(a), dag.VertexName(a_d))
g.Connect(&DestroyEdge{S: a, T: a_d})
}
}
// This is strange but is the easiest way to get the dependencies
// of a node that is being destroyed. We use another graph to make sure
// the resource is in the graph and ask for references. We have to do this
// because the node that is being destroyed may NOT be in the graph.
//
// Example: resource A is force new, then destroy A AND create A are
// in the graph. BUT if resource A is just pure destroy, then only
// destroy A is in the graph, and create A is not.
steps := []GraphTransformer{
&AttachResourceConfigTransformer{Module: t.Module},
&AttachStateTransformer{State: t.State},
}
// Go through all the nodes being destroyed and create a graph.
// The resulting graph is only of things being CREATED. For example,
// following our example, the resulting graph would be:
//
// A, B (with no edges)
//
var tempG Graph
for d, _ := range destroyers {
// d is what is being destroyed. We parse the resource address
// which it came from it is a panic if this fails.
addr, err := ParseResourceAddress(d)
if err != nil {
panic(err)
}
// This part is a little bit weird but is the best way to
// find the dependencies we need to: build a graph and use the
// attach config and state transformers then ask for references.
node := &NodeAbstractResource{Addr: addr}
tempG.Add(node)
}
// Run the graph transforms so we have the information we need to
// build references.
for _, s := range steps {
if err := s.Transform(&tempG); err != nil {
return err
}
}
// Create a reference map for easy lookup
refMap := NewReferenceMap(tempG.Vertices())
// Go through all the nodes in the graph and determine what they
// depend on.
for _, v := range tempG.Vertices() {
// Find all the references
refs, _ := refMap.References(v)
log.Printf(
"[TRACE] DestroyEdgeTransformer: creation node %q references %v",
dag.VertexName(v), refs)
// If we have no references, then we won't need to do anything
if len(refs) == 0 {
continue
}
// Get the destroy node for this. In the example of our struct,
// we are currently at B and we're looking for B_d.
rn, ok := v.(GraphNodeResource)
if !ok {
continue
}
addr := rn.ResourceAddr()
if addr == nil {
continue
}
dns := destroyers[addr.String()]
// We have dependencies, check if any are being destroyed
// to build the list of things that we must depend on!
//
// In the example of the struct, if we have:
//
// B_d => A_d => A => B
//
// Then at this point in the algorithm we started with B_d,
// we built B (to get dependencies), and we found A. We're now looking
// to see if A_d exists.
var depDestroyers []dag.Vertex
for _, v := range refs {
rn, ok := v.(GraphNodeResource)
if !ok {
continue
}
addr := rn.ResourceAddr()
if addr == nil {
continue
}
key := addr.String()
if ds, ok := destroyers[key]; ok {
for _, d := range ds {
depDestroyers = append(depDestroyers, d.(dag.Vertex))
log.Printf(
"[TRACE] DestroyEdgeTransformer: destruction of %q depends on %s",
key, dag.VertexName(d))
}
}
}
// Go through and make the connections. Use the variable
// names "a_d" and "b_d" to reference our example.
for _, a_d := range dns {
for _, b_d := range depDestroyers {
if b_d != a_d {
g.Connect(dag.BasicEdge(b_d, a_d))
}
}
}
}
return nil
}
// compileImports需要一个文件，load所有的imports,然后合并到文件中
func compileImports(root *File, importOpts *compileImportOpts, opts *CompileOpts) error {
// 如果没有imports，直接短路(返回)
if len(root.Imports) == 0 {
return nil
}
// 把它们放入变量，以便我们可以更早的引用
//storage := importOpts.Storage
//cache := importOpts.Cache
//cacheLock := importOpts.CacheLock
// A graph is used to track for cycles
var graphLock sync.Mutex
graph := new(dag.AcyclicGraph)
graph.Add("root")
// 自从我们平行的执行导入，在同一时间会有多个error发生
// 我们使用multierror lock和跟踪error
var resultErr error
var resultErrLock sync.Mutex
// Forward declarations for some nested functions we use. The docs
// for these functions are above each.
var importSingle func(parent string, f *File) bool
var downloadSingle func(string, *sync.WaitGroup, *sync.Mutex, []*File, int)
// importSingle is responsible for kicking off the imports and merging
// them for a single file. This will return true on success, false on
// failure. On failure, it is expected that any errors are appended to
// resultErr.
importSingle = func(parent string, f *File) bool {
var wg sync.WaitGroup
// 构建文件列表，后面将合并
var mergeLock sync.Mutex
merge := make([]*File, len(f.Imports))
// 通过导入并开始处理下载
for idx, i := range f.Imports {
source, err := getter.Detect(i.Source, filepath.Dir(f.Path), getter.Detectors)
if err != nil {
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"获取import源错误: %s", err))
return false
}
// Add this to the graph and check now if there are cycles
graphLock.Lock()
graph.Add(source)
graph.Connect(dag.BasicEdge(parent, source))
cycles := graph.Cycles()
graphLock.Unlock()
if len(cycles) > 0 {
for _, cycle := range cycles {
names := make([]string, len(cycle))
for i, v := range cycle {
names[i] = dag.VertexName(v)
}
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"Cycle found: %s", strings.Join(names, ", ")))
return false
}
}
wg.Add(1)
go downloadSingle(source, &wg, &mergeLock, merge, idx)
}
// Wait for completion
wg.Wait()
// Go through the merge list and look for any nil entries, which
// means that download failed. In that case, return immediately.
// We assume any errors were put into resultErr.
for _, importF := range merge {
if importF == nil {
return false
}
}
for _, importF := range merge {
// We need to copy importF here so that we don't poison
// the cache by modifying the same pointer.
importFCopy := *importF
importF = &importFCopy
source := importF.ID
importF.ID = ""
importF.Path = ""
// Merge it into our file!
if err := f.Merge(importF); err != nil {
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"合并import错误 %s : %s", source, err))
return false
}
}
return true
}
// downloadSingle is used to download a single import and parse the
// Appfile. This is a separate function because it is generally run
// in a goroutine so we can parallelize grabbing the imports.
downloadSingle = func(source string, wg *sync.WaitGroup, l *sync.Mutex, result []*File, idx int) {}
importSingle("root", root)
return resultErr
}
func (t *CreateBeforeDestroyTransformer) Transform(g *Graph) error {
// We "stage" the edge connections/destroys in these slices so that
// while we're doing the edge transformations (transpositions) in
// the graph, we're not affecting future edge transpositions. These
// slices let us stage ALL the changes that WILL happen so that all
// of the transformations happen atomically.
var connect, destroy []dag.Edge
for _, v := range g.Vertices() {
// We only care to use the destroy nodes
dn, ok := v.(GraphNodeDestroy)
if !ok {
continue
}
// If the node doesn't need to create before destroy, then continue
if !dn.CreateBeforeDestroy() {
if noCreateBeforeDestroyAncestors(g, dn) {
continue
}
// PURPOSELY HACKY FIX SINCE THIS TRANSFORM IS DEPRECATED.
// This is a hacky way to fix GH-10439. For a detailed description
// of the fix, see CBDEdgeTransformer, which is the equivalent
// transform used by the new graphs.
//
// This transform is deprecated because it is only used by the
// old graphs which are going to be removed.
var update *config.Resource
if dn, ok := v.(*graphNodeResourceDestroy); ok {
update = dn.Original.Resource
}
if dn, ok := v.(*graphNodeResourceDestroyFlat); ok {
update = dn.Original.Resource
}
if update != nil {
update.Lifecycle.CreateBeforeDestroy = true
}
}
// Get the creation side of this node
cn := dn.CreateNode()
// Take all the things which depend on the creation node and
// make them dependencies on the destruction. Clarifying this
// with an example: if you have a web server and a load balancer
// and the load balancer depends on the web server, then when we
// do a create before destroy, we want to make sure the steps are:
//
// 1.) Create new web server
// 2.) Update load balancer
// 3.) Delete old web server
//
// This ensures that.
for _, sourceRaw := range g.UpEdges(cn).List() {
source := sourceRaw.(dag.Vertex)
// If the graph has a "root" node (one added by a RootTransformer and not
// just a resource that happens to have no ancestors), we don't want to
// add any edges to it, because then it ceases to be a root.
if _, ok := source.(graphNodeRoot); ok {
continue
}
connect = append(connect, dag.BasicEdge(dn, source))
}
// Swap the edge so that the destroy depends on the creation
// happening...
connect = append(connect, dag.BasicEdge(dn, cn))
destroy = append(destroy, dag.BasicEdge(cn, dn))
}
for _, edge := range connect {
g.Connect(edge)
}
for _, edge := range destroy {
g.RemoveEdge(edge)
}
return nil
}
func (t *DestroyTransformer) Transform(g *Graph) error {
var connect, remove []dag.Edge
nodeToCn := make(map[dag.Vertex]dag.Vertex, len(g.Vertices()))
nodeToDn := make(map[dag.Vertex]dag.Vertex, len(g.Vertices()))
for _, v := range g.Vertices() {
// If it is not a destroyable, we don't care
cn, ok := v.(GraphNodeDestroyable)
if !ok {
continue
}
// Grab the destroy side of the node and connect it through
n := cn.DestroyNode()
if n == nil {
continue
}
// Store it
nodeToCn[n] = cn
nodeToDn[cn] = n
// If the creation node is equal to the destroy node, then
// don't do any of the edge jump rope below.
if n.(interface{}) == cn.(interface{}) {
continue
}
// Add it to the graph
g.Add(n)
// Inherit all the edges from the old node
downEdges := g.DownEdges(v).List()
for _, edgeRaw := range downEdges {
// If this thing specifically requests to not be depended on
// by destroy nodes, then don't.
if i, ok := edgeRaw.(GraphNodeDestroyEdgeInclude); ok &&
!i.DestroyEdgeInclude(v) {
continue
}
g.Connect(dag.BasicEdge(n, edgeRaw.(dag.Vertex)))
}
// Add a new edge to connect the node to be created to
// the destroy node.
connect = append(connect, dag.BasicEdge(v, n))
}
// Go through the nodes we added and determine if they depend
// on any nodes with a destroy node. If so, depend on that instead.
for n, _ := range nodeToCn {
for _, downRaw := range g.DownEdges(n).List() {
target := downRaw.(dag.Vertex)
cn2, ok := target.(GraphNodeDestroyable)
if !ok {
continue
}
newTarget := nodeToDn[cn2]
if newTarget == nil {
continue
}
// Make the new edge and transpose
connect = append(connect, dag.BasicEdge(newTarget, n))
// Remove the old edge
remove = append(remove, dag.BasicEdge(n, target))
}
}
// Atomatically add/remove the edges
for _, e := range connect {
g.Connect(e)
}
for _, e := range remove {
g.RemoveEdge(e)
}
return nil
}
func (t *DestroyTransformer) transform(
g *Graph, mode GraphNodeDestroyMode) ([]dag.Edge, []dag.Edge, error) {
var connect, remove []dag.Edge
nodeToCn := make(map[dag.Vertex]dag.Vertex, len(g.Vertices()))
nodeToDn := make(map[dag.Vertex]dag.Vertex, len(g.Vertices()))
for _, v := range g.Vertices() {
// If it is not a destroyable, we don't care
cn, ok := v.(GraphNodeDestroyable)
if !ok {
continue
}
// Grab the destroy side of the node and connect it through
n := cn.DestroyNode(mode)
if n == nil {
continue
}
// Store it
nodeToCn[n] = cn
nodeToDn[cn] = n
// Add it to the graph
g.Add(n)
// Inherit all the edges from the old node
downEdges := g.DownEdges(v).List()
for _, edgeRaw := range downEdges {
g.Connect(dag.BasicEdge(n, edgeRaw.(dag.Vertex)))
}
// Add a new edge to connect the node to be created to
// the destroy node.
connect = append(connect, dag.BasicEdge(v, n))
}
// Go through the nodes we added and determine if they depend
// on any nodes with a destroy node. If so, depend on that instead.
for n, _ := range nodeToCn {
for _, downRaw := range g.DownEdges(n).List() {
target := downRaw.(dag.Vertex)
cn2, ok := target.(GraphNodeDestroyable)
if !ok {
continue
}
newTarget := nodeToDn[cn2]
if newTarget == nil {
continue
}
// Make the new edge and transpose
connect = append(connect, dag.BasicEdge(newTarget, n))
// Remove the old edge
remove = append(remove, dag.BasicEdge(n, target))
}
}
return connect, remove, nil
}
func (c *Compiler) compileDependencies(root *CompiledGraphVertex, graph *dag.AcyclicGraph) error {
// For easier reference below
storage := c.depStorage
// Make a map to keep track of the dep source to vertex mapping
vertexMap := make(map[string]*CompiledGraphVertex)
// Store ourselves in the map
key, err := getter.Detect(
".", filepath.Dir(root.File.Path),
getter.Detectors)
if err != nil {
return err
}
vertexMap[key] = root
// Make a queue for the other vertices we need to still get
// dependencies for. We arbitrarily make the cap for this slice
// 30, since that is a ton of dependencies and we don't expect the
// average case to have more than this.
queue := make([]*CompiledGraphVertex, 1, 30)
queue[0] = root
// While we still have dependencies to get, continue loading them.
// TODO: parallelize
for len(queue) > 0 {
var current *CompiledGraphVertex
current, queue = queue[len(queue)-1], queue[:len(queue)-1]
log.Printf("[DEBUG] compiling dependencies for: %s", current.Name())
for _, dep := range current.File.Application.Dependencies {
key, err := getter.Detect(
dep.Source, filepath.Dir(current.File.Path),
getter.Detectors)
if err != nil {
return fmt.Errorf(
"Error loading source: %s", err)
}
vertex := vertexMap[key]
if vertex == nil {
log.Printf("[DEBUG] loading dependency: %s", key)
// Call the callback if we have one
if c.opts.Callback != nil {
c.opts.Callback(&CompileEventDep{
Source: key,
})
}
// Download the dependency
if err := storage.Get(key, key, true); err != nil {
return err
}
dir, _, err := storage.Dir(key)
if err != nil {
return err
}
// Parse the Appfile if it exists
var f *File
appfilePath := filepath.Join(dir, "Appfile")
_, err = os.Stat(appfilePath)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf(
"Error parsing Appfile in %s: %s", key, err)
}
if err == nil {
f, err = ParseFile(appfilePath)
if err != nil {
return fmt.Errorf(
"Error parsing Appfile in %s: %s", key, err)
}
// Realize all the imports for this file
if err := c.compileImports(f); err != nil {
return err
}
}
// Do any additional loading if we have a loader
if c.opts.Loader != nil {
f, err = c.opts.Loader(f, dir)
if err != nil {
return fmt.Errorf(
"Error loading Appfile in %s: %s", key, err)
}
}
// Set the source
f.Source = key
// If it doesn't have an otto ID then we can't do anything
hasID, err := f.hasID()
if err != nil {
return fmt.Errorf(
"Error checking for ID file for Appfile in %s: %s",
key, err)
}
if !hasID {
return fmt.Errorf(
"Dependency '%s' doesn't have an Otto ID yet!\n\n"+
"An Otto ID is generated on the first compilation of the Appfile.\n"+
"It is a globally unique ID that is used to track the application\n"+
"across multiple deploys. It is required for the application to be\n"+
"used as a dependency. To fix this, check out that application and\n"+
"compile the Appfile with `otto compile` once. Make sure you commit\n"+
"the .ottoid file into version control, and then try this command\n"+
"again.",
key)
}
// We merge the root infrastructure choice upwards to
// all dependencies.
f.Infrastructure = root.File.Infrastructure
if root.File.Project != nil {
if f.Project == nil {
f.Project = new(Project)
}
f.Project.Infrastructure = root.File.Project.Infrastructure
}
// Build the vertex for this
vertex = &CompiledGraphVertex{
File: f,
Dir: dir,
NameValue: f.Application.Name,
}
// Add the vertex since it is new, store the mapping, and
// queue it to be loaded later.
graph.Add(vertex)
vertexMap[key] = vertex
queue = append(queue, vertex)
}
// Connect the dependencies
graph.Connect(dag.BasicEdge(current, vertex))
}
}
return nil
}
func (t *CBDEdgeTransformer) Transform(g *Graph) error {
log.Printf("[TRACE] CBDEdgeTransformer: Beginning CBD transformation...")
// Go through and reverse any destroy edges
destroyMap := make(map[string][]dag.Vertex)
for _, v := range g.Vertices() {
dn, ok := v.(GraphNodeDestroyerCBD)
if !ok {
continue
}
if !dn.CreateBeforeDestroy() {
continue
}
// Find the destroy edge. There should only be one.
for _, e := range g.EdgesTo(v) {
// Not a destroy edge, ignore it
de, ok := e.(*DestroyEdge)
if !ok {
continue
}
log.Printf("[TRACE] CBDEdgeTransformer: inverting edge: %s => %s",
dag.VertexName(de.Source()), dag.VertexName(de.Target()))
// Found it! Invert.
g.RemoveEdge(de)
g.Connect(&DestroyEdge{S: de.Target(), T: de.Source()})
}
// Add this to the list of nodes that we need to fix up
// the edges for (step 2 above in the docs).
key := dn.DestroyAddr().String()
destroyMap[key] = append(destroyMap[key], v)
}
// If we have no CBD nodes, then our work here is done
if len(destroyMap) == 0 {
return nil
}
// We have CBD nodes. We now have to move on to the much more difficult
// task of connecting dependencies of the creation side of the destroy
// to the destruction node. The easiest way to explain this is an example:
//
// Given a pre-destroy dependence of: A => B
// And A has CBD set.
//
// The resulting graph should be: A => B => A_d
//
// They key here is that B happens before A is destroyed. This is to
// facilitate the primary purpose for CBD: making sure that downstreams
// are properly updated to avoid downtime before the resource is destroyed.
//
// We can't trust that the resource being destroyed or anything that
// depends on it is actually in our current graph so we make a new
// graph in order to determine those dependencies and add them in.
log.Printf("[TRACE] CBDEdgeTransformer: building graph to find dependencies...")
depMap, err := t.depMap(destroyMap)
if err != nil {
return err
}
// We now have the mapping of resource addresses to the destroy
// nodes they need to depend on. We now go through our own vertices to
// find any matching these addresses and make the connection.
for _, v := range g.Vertices() {
// We're looking for creators
rn, ok := v.(GraphNodeCreator)
if !ok {
continue
}
// Get the address
addr := rn.CreateAddr()
key := addr.String()
// If there is nothing this resource should depend on, ignore it
dns, ok := depMap[key]
if !ok {
continue
}
// We have nodes! Make the connection
for _, dn := range dns {
log.Printf("[TRACE] CBDEdgeTransformer: destroy depends on dependence: %s => %s",
dag.VertexName(dn), dag.VertexName(v))
g.Connect(dag.BasicEdge(dn, v))
}
}
return nil
}
func (l *Layered) graph(db *bolt.DB) (*dag.AcyclicGraph, error) {
graph := new(dag.AcyclicGraph)
graph.Add("root")
// First, add all the layers
layers := make(map[string]*layerVertex)
err := db.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket(boltLayersBucket)
return bucket.ForEach(func(k, data []byte) error {
var v layerVertex
if err := l.structRead(&v, data); err != nil {
return err
}
// Add this layer to the graph
graph.Add(&v)
// Store the mapping for later
layers[v.Layer.ID] = &v
return nil
})
})
if err != nil {
return nil, err
}
// Next, connect the layers
err = db.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket(boltEdgesBucket)
return bucket.ForEach(func(k, data []byte) error {
from := layers[string(k)]
to := layers[string(data)]
if from != nil && to != nil {
graph.Connect(dag.BasicEdge(from, to))
}
return nil
})
})
if err != nil {
return nil, err
}
// Finally, add and connect all the envs
err = db.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket(boltEnvsBucket)
return bucket.ForEach(func(k, data []byte) error {
key := fmt.Sprintf("env-%s", string(k))
graph.Add(key)
// Connect the env to the layer it depends on
to := &layerVertex{Layer: &Layer{ID: string(data)}}
graph.Connect(dag.BasicEdge(key, to))
// Connect the root to the environment that is active
graph.Connect(dag.BasicEdge("root", key))
return nil
})
})
if err != nil {
return nil, err
}
return graph, nil
}
// compileImports takes a File, loads all the imports, and merges them
// into the File.
func (c *Compiler) compileImports(root *File) error {
// If we have no imports, short-circuit the whole thing
if len(root.Imports) == 0 {
return nil
}
// Pull these out into variables so they're easier to reference
storage := c.importStorage
cache := c.importCache
cacheLock := &c.importLock
// A graph is used to track for cycles
var graphLock sync.Mutex
graph := new(dag.AcyclicGraph)
graph.Add("root")
// Since we run the import in parallel, multiple errors can happen
// at the same time. We use multierror and a lock to keep track of errors.
var resultErr error
var resultErrLock sync.Mutex
// Forward declarations for some nested functions we use. The docs
// for these functions are above each.
var importSingle func(parent string, f *File) bool
var downloadSingle func(string, *sync.WaitGroup, *sync.Mutex, []*File, int)
// importSingle is responsible for kicking off the imports and merging
// them for a single file. This will return true on success, false on
// failure. On failure, it is expected that any errors are appended to
// resultErr.
importSingle = func(parent string, f *File) bool {
var wg sync.WaitGroup
// Build the list of files we'll merge later
var mergeLock sync.Mutex
merge := make([]*File, len(f.Imports))
// Go through the imports and kick off the download
for idx, i := range f.Imports {
source, err := getter.Detect(
i.Source, filepath.Dir(f.Path),
getter.Detectors)
if err != nil {
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"Error loading import source: %s", err))
return false
}
// Add this to the graph and check now if there are cycles
graphLock.Lock()
graph.Add(source)
graph.Connect(dag.BasicEdge(parent, source))
cycles := graph.Cycles()
graphLock.Unlock()
if len(cycles) > 0 {
for _, cycle := range cycles {
names := make([]string, len(cycle))
for i, v := range cycle {
names[i] = dag.VertexName(v)
}
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"Cycle found: %s", strings.Join(names, ", ")))
return false
}
}
wg.Add(1)
go downloadSingle(source, &wg, &mergeLock, merge, idx)
}
// Wait for completion
wg.Wait()
// Go through the merge list and look for any nil entries, which
// means that download failed. In that case, return immediately.
// We assume any errors were put into resultErr.
for _, importF := range merge {
if importF == nil {
return false
}
}
for _, importF := range merge {
// We need to copy importF here so that we don't poison
// the cache by modifying the same pointer.
importFCopy := *importF
importF = &importFCopy
source := importF.ID
importF.ID = ""
importF.Path = ""
// Merge it into our file!
if err := f.Merge(importF); err != nil {
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"Error merging import %s: %s", source, err))
return false
}
}
return true
}
// downloadSingle is used to download a single import and parse the
// Appfile. This is a separate function because it is generally run
// in a goroutine so we can parallelize grabbing the imports.
downloadSingle = func(source string, wg *sync.WaitGroup, l *sync.Mutex, result []*File, idx int) {
defer wg.Done()
// Read from the cache if we have it
cacheLock.Lock()
cached, ok := cache[source]
cacheLock.Unlock()
if ok {
log.Printf("[DEBUG] cache hit on import: %s", source)
l.Lock()
defer l.Unlock()
result[idx] = cached
return
}
// Call the callback if we have one
log.Printf("[DEBUG] loading import: %s", source)
if c.opts.Callback != nil {
c.opts.Callback(&CompileEventImport{
Source: source,
})
}
// Download the dependency
if err := storage.Get(source, source, true); err != nil {
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"Error loading import source: %s", err))
return
}
dir, _, err := storage.Dir(source)
if err != nil {
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"Error loading import source: %s", err))
return
}
// Parse the Appfile
importF, err := ParseFile(filepath.Join(dir, "Appfile"))
if err != nil {
resultErrLock.Lock()
defer resultErrLock.Unlock()
resultErr = multierror.Append(resultErr, fmt.Errorf(
"Error parsing Appfile in %s: %s", source, err))
return
}
// We use the ID to store the source, but we clear it
// when we actually merge.
importF.ID = source
// Import the imports in this
if !importSingle(source, importF) {
return
}
// Once we're done, acquire the lock and write it
l.Lock()
result[idx] = importF
l.Unlock()
// Write this into the cache.
cacheLock.Lock()
cache[source] = importF
cacheLock.Unlock()
}
importSingle("root", root)
return resultErr
}