func TestPruneNoopTransformer(t *testing.T) {
	g := Graph{Path: RootModulePath}

	a := &testGraphNodeNoop{NameValue: "A"}
	b := &testGraphNodeNoop{NameValue: "B", Value: true}
	c := &testGraphNodeNoop{NameValue: "C"}

	g.Add(a)
	g.Add(b)
	g.Add(c)
	g.Connect(dag.BasicEdge(a, b))
	g.Connect(dag.BasicEdge(b, c))

	{
		tf := &PruneNoopTransformer{}
		if err := tf.Transform(&g); err != nil {
			t.Fatalf("err: %s", err)
		}
	}

	actual := strings.TrimSpace(g.String())
	expected := strings.TrimSpace(testTransformPruneNoopStr)
	if actual != expected {
		t.Fatalf("bad:\n\n%s", actual)
	}
}
func TestVertexTransformer(t *testing.T) {
	var g Graph
	g.Add(1)
	g.Add(2)
	g.Add(3)
	g.Connect(dag.BasicEdge(1, 2))
	g.Connect(dag.BasicEdge(2, 3))

	{
		tf := &VertexTransformer{
			Transforms: []GraphVertexTransformer{
				&testVertexTransform{Source: 2, Target: 42},
			},
		}
		if err := tf.Transform(&g); err != nil {
			t.Fatalf("err: %s", err)
		}
	}

	actual := strings.TrimSpace(g.String())
	expected := strings.TrimSpace(testVertexTransformerStr)
	if actual != expected {
		t.Fatalf("bad: %s", actual)
	}
}
Ejemplo n.º 3
0
func (t *CreateBeforeDestroyTransformer) Transform(g *Graph) error {
	// We "stage" the edge connections/destroys in these slices so that
	// while we're doing the edge transformations (transpositions) in
	// the graph, we're not affecting future edge transpositions. These
	// slices let us stage ALL the changes that WILL happen so that all
	// of the transformations happen atomically.
	var connect, destroy []dag.Edge

	for _, v := range g.Vertices() {
		// We only care to use the destroy nodes
		dn, ok := v.(GraphNodeDestroy)
		if !ok {
			continue
		}

		// If the node doesn't need to create before destroy, then continue
		if !dn.CreateBeforeDestroy() {
			continue
		}

		// Get the creation side of this node
		cn := dn.CreateNode()

		// Take all the things which depend on the creation node and
		// make them dependencies on the destruction. Clarifying this
		// with an example: if you have a web server and a load balancer
		// and the load balancer depends on the web server, then when we
		// do a create before destroy, we want to make sure the steps are:
		//
		// 1.) Create new web server
		// 2.) Update load balancer
		// 3.) Delete old web server
		//
		// This ensures that.
		for _, sourceRaw := range g.UpEdges(cn).List() {
			source := sourceRaw.(dag.Vertex)
			connect = append(connect, dag.BasicEdge(dn, source))
		}

		// Swap the edge so that the destroy depends on the creation
		// happening...
		connect = append(connect, dag.BasicEdge(dn, cn))
		destroy = append(destroy, dag.BasicEdge(cn, dn))
	}

	for _, edge := range connect {
		g.Connect(edge)
	}
	for _, edge := range destroy {
		g.RemoveEdge(edge)
	}

	return nil
}
Ejemplo n.º 4
0
func (t *Tree) buildProviderAliasGraph(g *dag.AcyclicGraph, parent dag.Vertex) {
	// Add all our defined aliases
	defined := make(map[string]struct{})
	for _, p := range t.config.ProviderConfigs {
		defined[p.FullName()] = struct{}{}
	}

	// Add all our used aliases
	used := make(map[string]struct{})
	for _, r := range t.config.Resources {
		if r.Provider != "" {
			used[r.Provider] = struct{}{}
		}
	}

	// Add it to the graph
	vertex := &providerAliasVertex{
		Path:    t.Path(),
		Defined: defined,
		Used:    used,
	}
	g.Add(vertex)

	// Connect to our parent if we have one
	if parent != nil {
		g.Connect(dag.BasicEdge(vertex, parent))
	}

	// Build all our children
	for _, c := range t.Children() {
		c.buildProviderAliasGraph(g, vertex)
	}
}
Ejemplo n.º 5
0
// ConnectFrom creates an edge by finding the source from a DependableName
// and connecting it to the specific vertex.
func (g *Graph) ConnectFrom(source string, target dag.Vertex) {
	g.once.Do(g.init)

	if source := g.dependableMap[source]; source != nil {
		g.Connect(dag.BasicEdge(source, target))
	}
}
Ejemplo n.º 6
0
func (t *ProxyTransformer) Transform(g *Graph) error {
	for _, v := range g.Vertices() {
		pn, ok := v.(GraphNodeProxy)
		if !ok {
			continue
		}

		// If we don't want to be proxies, don't do it
		if !pn.Proxy() {
			continue
		}

		// Connect all the things that depend on this to things that
		// we depend on as the proxy. See docs for GraphNodeProxy for
		// a visual explanation.
		for _, s := range g.UpEdges(v).List() {
			for _, t := range g.DownEdges(v).List() {
				g.Connect(GraphProxyEdge{
					Edge: dag.BasicEdge(s, t),
				})
			}
		}
	}

	return nil
}
Ejemplo n.º 7
0
func (c *Compiled) UnmarshalJSON(data []byte) error {
	var raw compiledJSON
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	c.File = raw.File
	c.Graph = new(dag.AcyclicGraph)
	for _, v := range raw.Vertices {
		c.Graph.Add(v)
	}
	for _, e := range raw.Edges {
		for a, b := range e {
			ai, err := strconv.ParseInt(a, 0, 0)
			if err != nil {
				return err
			}

			bi, err := strconv.ParseInt(b, 0, 0)
			if err != nil {
				return err
			}

			c.Graph.Connect(dag.BasicEdge(raw.Vertices[ai], raw.Vertices[bi]))
		}
	}

	return nil
}
func TestExpandTransform(t *testing.T) {
	var g Graph
	g.Add(1)
	g.Add(2)
	g.Connect(dag.BasicEdge(1, 2))

	tf := &ExpandTransform{}
	out, err := tf.Transform(&testExpandable{
		Result: &g,
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	sn, ok := out.(GraphNodeSubgraph)
	if !ok {
		t.Fatalf("not subgraph: %#v", out)
	}

	actual := strings.TrimSpace(sn.Subgraph().String())
	expected := strings.TrimSpace(testExpandTransformStr)
	if actual != expected {
		t.Fatalf("bad: %s", actual)
	}
}
Ejemplo n.º 9
0
func (t *PruneNoopTransformer) Transform(g *Graph) error {
	// Find the leaves.
	leaves := make([]dag.Vertex, 0, 10)
	for _, v := range g.Vertices() {
		if g.DownEdges(v).Len() == 0 {
			leaves = append(leaves, v)
		}
	}

	// Do a depth first walk from the leaves and remove things.
	return g.ReverseDepthFirstWalk(leaves, func(v dag.Vertex, depth int) error {
		// We need a prunable
		pn, ok := v.(GraphNodeNoopPrunable)
		if !ok {
			return nil
		}

		// Start building the noop opts
		path := g.Path
		if pn, ok := v.(GraphNodeSubPath); ok {
			path = pn.Path()
		}

		var modDiff *ModuleDiff
		var modState *ModuleState
		if t.Diff != nil {
			modDiff = t.Diff.ModuleByPath(path)
		}
		if t.State != nil {
			modState = t.State.ModuleByPath(path)
		}

		// Determine if its a noop. If it isn't, just return
		noop := pn.Noop(&NoopOpts{
			Graph:    g,
			Vertex:   v,
			Diff:     t.Diff,
			State:    t.State,
			ModDiff:  modDiff,
			ModState: modState,
		})
		if !noop {
			return nil
		}

		// It is a noop! We first preserve edges.
		up := g.UpEdges(v).List()
		for _, downV := range g.DownEdges(v).List() {
			for _, upV := range up {
				g.Connect(dag.BasicEdge(upV, downV))
			}
		}

		// Then remove it
		g.Remove(v)

		return nil
	})
}
Ejemplo n.º 10
0
func TestDebugJSON2Dot(t *testing.T) {
	// create the graph JSON output
	logFile, err := ioutil.TempFile("", "tf")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(logFile.Name())

	var g dag.Graph
	g.SetDebugWriter(logFile)

	g.Add(1)
	g.Add(2)
	g.Add(3)
	g.Connect(dag.BasicEdge(1, 2))
	g.Connect(dag.BasicEdge(2, 3))

	ui := new(cli.MockUi)
	c := &DebugJSON2DotCommand{
		Meta: Meta{
			ContextOpts: testCtxConfig(testProvider()),
			Ui:          ui,
		},
	}

	args := []string{
		logFile.Name(),
	}
	if code := c.Run(args); code != 0 {
		t.Fatalf("bad: \n%s", ui.ErrorWriter.String())
	}

	output := ui.OutputWriter.String()
	if !strings.HasPrefix(output, "digraph {") {
		t.Fatalf("doesn't look like digraph: %s", output)
	}

	if !strings.Contains(output, `subgraph "root" {`) {
		t.Fatalf("doesn't contains root subgraph: %s", output)
	}
}
func (t *ParentProviderTransformer) Transform(g *Graph) error {
	// Make a mapping of path to dag.Vertex, where path is: "path.name"
	m := make(map[string]dag.Vertex)

	// Also create a map that maps a provider to its parent
	parentMap := make(map[dag.Vertex]string)
	for _, raw := range g.Vertices() {
		// If it is the flat version, then make it the non-flat version.
		// We eventually want to get rid of the flat version entirely so
		// this is a stop-gap while it still exists.
		var v dag.Vertex = raw
		if f, ok := v.(*graphNodeProviderFlat); ok {
			v = f.graphNodeProvider
		}

		// Only care about providers
		pn, ok := v.(GraphNodeProvider)
		if !ok || pn.ProviderName() == "" {
			continue
		}

		// Also require a subpath, if there is no subpath then we
		// just totally ignore it. The expectation of this transform is
		// that it is used with a graph builder that is already flattened.
		var path []string
		if pn, ok := raw.(GraphNodeSubPath); ok {
			path = pn.Path()
		}
		path = normalizeModulePath(path)

		// Build the key with path.name i.e. "child.subchild.aws"
		key := fmt.Sprintf("%s.%s", strings.Join(path, "."), pn.ProviderName())
		m[key] = raw

		// Determine the parent if we're non-root. This is length 1 since
		// the 0 index should be "root" since we normalize above.
		if len(path) > 1 {
			path = path[:len(path)-1]
			key := fmt.Sprintf("%s.%s", strings.Join(path, "."), pn.ProviderName())
			parentMap[raw] = key
		}
	}

	// Connect!
	for v, key := range parentMap {
		if parent, ok := m[key]; ok {
			g.Connect(dag.BasicEdge(v, parent))
		}
	}

	return nil
}
func TestProxyTransformer(t *testing.T) {
	var g Graph
	proxy := &testNodeProxy{NameValue: "proxy"}
	g.Add("A")
	g.Add("C")
	g.Add(proxy)
	g.Connect(dag.BasicEdge("A", proxy))
	g.Connect(dag.BasicEdge(proxy, "C"))

	{
		tf := &ProxyTransformer{}
		if err := tf.Transform(&g); err != nil {
			t.Fatalf("err: %s", err)
		}
	}

	actual := strings.TrimSpace(g.String())
	expected := strings.TrimSpace(testProxyTransformStr)
	if actual != expected {
		t.Fatalf("bad: %s", actual)
	}
}
func (t *CloseProviderTransformer) Transform(g *Graph) error {
	pm := providerVertexMap(g)
	cpm := closeProviderVertexMap(g)
	var err error
	for _, v := range g.Vertices() {
		if pv, ok := v.(GraphNodeProviderConsumer); ok {
			for _, p := range pv.ProvidedBy() {
				key := p
				source := cpm[key]

				if source == nil {
					// Create a new graphNodeCloseProvider and add it to the graph
					source = &graphNodeCloseProvider{ProviderNameValue: p}
					g.Add(source)

					// Close node needs to depend on provider
					provider, ok := pm[key]
					if !ok {
						err = multierror.Append(err, fmt.Errorf(
							"%s: provider %s couldn't be found for closing",
							dag.VertexName(v), p))
						continue
					}
					g.Connect(dag.BasicEdge(source, provider))

					// Make sure we also add the new graphNodeCloseProvider to the map
					// so we don't create and add any duplicate graphNodeCloseProviders.
					cpm[key] = source
				}

				// Close node depends on all nodes provided by the provider
				g.Connect(dag.BasicEdge(source, v))
			}
		}
	}

	return err
}
Ejemplo n.º 14
0
// ConnectTo connects a vertex to a raw string of targets that are the
// result of DependableName, and returns the list of targets that are missing.
func (g *Graph) ConnectTo(v dag.Vertex, targets []string) []string {
	g.once.Do(g.init)

	var missing []string
	for _, t := range targets {
		if dest := g.dependableMap[t]; dest != nil {
			g.Connect(dag.BasicEdge(v, dest))
		} else {
			missing = append(missing, t)
		}
	}

	return missing
}
Ejemplo n.º 15
0
func (t *CountBoundaryTransformer) Transform(g *Graph) error {
	node := &NodeCountBoundary{}
	g.Add(node)

	// Depends on everything
	for _, v := range g.Vertices() {
		// Don't connect to ourselves
		if v == node {
			continue
		}

		// Connect!
		g.Connect(dag.BasicEdge(node, v))
	}

	return nil
}
Ejemplo n.º 16
0
func (t *ModuleInputTransformer) Transform(g *Graph) error {
	// Create the node
	n := &graphNodeModuleInput{Variables: t.Variables}

	// Add it to the graph
	g.Add(n)

	// Connect the inputs to the bottom of the graph so that it happens
	// first.
	for _, v := range g.Vertices() {
		if v == n {
			continue
		}

		if g.DownEdges(v).Len() == 0 {
			g.Connect(dag.BasicEdge(v, n))
		}
	}

	return nil
}
Ejemplo n.º 17
0
func (t *ProvisionerTransformer) Transform(g *Graph) error {
	// Go through the other nodes and match them to provisioners they need
	var err error
	m := provisionerVertexMap(g)
	for _, v := range g.Vertices() {
		if pv, ok := v.(GraphNodeProvisionerConsumer); ok {
			for _, p := range pv.ProvisionedBy() {
				if m[p] == nil {
					err = multierror.Append(err, fmt.Errorf(
						"%s: provisioner %s couldn't be found",
						dag.VertexName(v), p))
					continue
				}

				g.Connect(dag.BasicEdge(v, m[p]))
			}
		}
	}

	return err
}
Ejemplo n.º 18
0
func (t *RootTransformer) Transform(g *Graph) error {
	// If we already have a good root, we're done
	if _, err := g.Root(); err == nil {
		return nil
	}

	// Add a root
	var root graphNodeRoot
	g.Add(root)

	// Connect the root to all the edges that need it
	for _, v := range g.Vertices() {
		if v == root {
			continue
		}

		if g.UpEdges(v).Len() == 0 {
			g.Connect(dag.BasicEdge(root, v))
		}
	}

	return nil
}
func (t *ProviderTransformer) Transform(g *Graph) error {
	// Go through the other nodes and match them to providers they need
	var err error
	m := providerVertexMap(g)
	for _, v := range g.Vertices() {
		if pv, ok := v.(GraphNodeProviderConsumer); ok {
			for _, p := range pv.ProvidedBy() {
				target := m[providerMapKey(p, pv)]
				if target == nil {
					println(fmt.Sprintf("%#v\n\n%#v", m, providerMapKey(p, pv)))
					err = multierror.Append(err, fmt.Errorf(
						"%s: provider %s couldn't be found",
						dag.VertexName(v), p))
					continue
				}

				g.Connect(dag.BasicEdge(v, target))
			}
		}
	}

	return err
}
Ejemplo n.º 20
0
func (t *ReferenceTransformer) Transform(g *Graph) error {
	// Build a reference map so we can efficiently look up the references
	vs := g.Vertices()
	m := NewReferenceMap(vs)

	// Find the things that reference things and connect them
	for _, v := range vs {
		parents, _ := m.References(v)
		parentsDbg := make([]string, len(parents))
		for i, v := range parents {
			parentsDbg[i] = dag.VertexName(v)
		}
		log.Printf(
			"[DEBUG] ReferenceTransformer: %q references: %v",
			dag.VertexName(v), parentsDbg)

		for _, parent := range parents {
			g.Connect(dag.BasicEdge(v, parent))
		}
	}

	return nil
}
Ejemplo n.º 21
0
func (t *CloseProviderTransformer) Transform(g *Graph) error {
	m := closeProviderVertexMap(g)
	for _, v := range g.Vertices() {
		if pv, ok := v.(GraphNodeProviderConsumer); ok {
			for _, p := range pv.ProvidedBy() {
				source := m[p]

				if source == nil {
					// Create a new graphNodeCloseProvider and add it to the graph
					source = &graphNodeCloseProvider{ProviderNameValue: p}
					g.Add(source)

					// Make sure we also add the new graphNodeCloseProvider to the map
					// so we don't create and add any duplicate graphNodeCloseProviders.
					m[p] = source
				}

				g.Connect(dag.BasicEdge(source, v))
			}
		}
	}

	return nil
}
Ejemplo n.º 22
0
func (t *DestroyEdgeTransformer) Transform(g *Graph) error {
	log.Printf("[TRACE] DestroyEdgeTransformer: Beginning destroy edge transformation...")

	// Build a map of what is being destroyed (by address string) to
	// the list of destroyers. In general there will only be one destroyer
	// but to make it more robust we support multiple.
	destroyers := make(map[string][]GraphNodeDestroyer)
	for _, v := range g.Vertices() {
		dn, ok := v.(GraphNodeDestroyer)
		if !ok {
			continue
		}

		addr := dn.DestroyAddr()
		if addr == nil {
			continue
		}

		key := addr.String()
		log.Printf(
			"[TRACE] DestroyEdgeTransformer: %s destroying %q",
			dag.VertexName(dn), key)
		destroyers[key] = append(destroyers[key], dn)
	}

	// If we aren't destroying anything, there will be no edges to make
	// so just exit early and avoid future work.
	if len(destroyers) == 0 {
		return nil
	}

	// Go through and connect creators to destroyers. Going along with
	// our example, this makes: A_d => A
	for _, v := range g.Vertices() {
		cn, ok := v.(GraphNodeCreator)
		if !ok {
			continue
		}

		addr := cn.CreateAddr()
		if addr == nil {
			continue
		}

		key := addr.String()
		ds := destroyers[key]
		if len(ds) == 0 {
			continue
		}

		for _, d := range ds {
			// For illustrating our example
			a_d := d.(dag.Vertex)
			a := v

			log.Printf(
				"[TRACE] DestroyEdgeTransformer: connecting creator/destroyer: %s, %s",
				dag.VertexName(a), dag.VertexName(a_d))

			g.Connect(&DestroyEdge{S: a, T: a_d})
		}
	}

	// This is strange but is the easiest way to get the dependencies
	// of a node that is being destroyed. We use another graph to make sure
	// the resource is in the graph and ask for references. We have to do this
	// because the node that is being destroyed may NOT be in the graph.
	//
	// Example: resource A is force new, then destroy A AND create A are
	// in the graph. BUT if resource A is just pure destroy, then only
	// destroy A is in the graph, and create A is not.
	steps := []GraphTransformer{
		&AttachResourceConfigTransformer{Module: t.Module},
		&AttachStateTransformer{State: t.State},
	}

	// Go through all the nodes being destroyed and create a graph.
	// The resulting graph is only of things being CREATED. For example,
	// following our example, the resulting graph would be:
	//
	//   A, B (with no edges)
	//
	var tempG Graph
	for d, _ := range destroyers {
		// d is what is being destroyed. We parse the resource address
		// which it came from it is a panic if this fails.
		addr, err := ParseResourceAddress(d)
		if err != nil {
			panic(err)
		}

		// This part is a little bit weird but is the best way to
		// find the dependencies we need to: build a graph and use the
		// attach config and state transformers then ask for references.
		node := &NodeAbstractResource{Addr: addr}
		tempG.Add(node)
	}

	// Run the graph transforms so we have the information we need to
	// build references.
	for _, s := range steps {
		if err := s.Transform(&tempG); err != nil {
			return err
		}
	}

	// Create a reference map for easy lookup
	refMap := NewReferenceMap(tempG.Vertices())

	// Go through all the nodes in the graph and determine what they
	// depend on.
	for _, v := range tempG.Vertices() {
		// Find all the references
		refs, _ := refMap.References(v)
		log.Printf(
			"[TRACE] DestroyEdgeTransformer: creation node %q references %v",
			dag.VertexName(v), refs)

		// If we have no references, then we won't need to do anything
		if len(refs) == 0 {
			continue
		}

		// Get the destroy node for this. In the example of our struct,
		// we are currently at B and we're looking for B_d.
		rn, ok := v.(GraphNodeResource)
		if !ok {
			continue
		}

		addr := rn.ResourceAddr()
		if addr == nil {
			continue
		}

		dns := destroyers[addr.String()]

		// We have dependencies, check if any are being destroyed
		// to build the list of things that we must depend on!
		//
		// In the example of the struct, if we have:
		//
		//   B_d => A_d => A => B
		//
		// Then at this point in the algorithm we started with B_d,
		// we built B (to get dependencies), and we found A. We're now looking
		// to see if A_d exists.
		var depDestroyers []dag.Vertex
		for _, v := range refs {
			rn, ok := v.(GraphNodeResource)
			if !ok {
				continue
			}

			addr := rn.ResourceAddr()
			if addr == nil {
				continue
			}

			key := addr.String()
			if ds, ok := destroyers[key]; ok {
				for _, d := range ds {
					depDestroyers = append(depDestroyers, d.(dag.Vertex))
					log.Printf(
						"[TRACE] DestroyEdgeTransformer: destruction of %q depends on %s",
						key, dag.VertexName(d))
				}
			}
		}

		// Go through and make the connections. Use the variable
		// names "a_d" and "b_d" to reference our example.
		for _, a_d := range dns {
			for _, b_d := range depDestroyers {
				if b_d != a_d {
					g.Connect(dag.BasicEdge(b_d, a_d))
				}
			}
		}
	}

	return nil
}
Ejemplo n.º 23
0
// compileImports需要一个文件,load所有的imports,然后合并到文件中
func compileImports(root *File, importOpts *compileImportOpts, opts *CompileOpts) error {
	// 如果没有imports,直接短路(返回)
	if len(root.Imports) == 0 {
		return nil
	}

	// 把它们放入变量,以便我们可以更早的引用
	//storage := importOpts.Storage
	//cache := importOpts.Cache
	//cacheLock := importOpts.CacheLock

	// A graph is used to track for cycles
	var graphLock sync.Mutex
	graph := new(dag.AcyclicGraph)
	graph.Add("root")

	// 自从我们平行的执行导入,在同一时间会有多个error发生
	// 我们使用multierror lock和跟踪error
	var resultErr error
	var resultErrLock sync.Mutex

	// Forward declarations for some nested functions we use. The docs
	// for these functions are above each.
	var importSingle func(parent string, f *File) bool
	var downloadSingle func(string, *sync.WaitGroup, *sync.Mutex, []*File, int)

	// importSingle is responsible for kicking off the imports and merging
	// them for a single file. This will return true on success, false on
	// failure. On failure, it is expected that any errors are appended to
	// resultErr.
	importSingle = func(parent string, f *File) bool {
		var wg sync.WaitGroup

		// 构建文件列表,后面将合并
		var mergeLock sync.Mutex
		merge := make([]*File, len(f.Imports))

		// 通过导入并开始处理下载
		for idx, i := range f.Imports {
			source, err := getter.Detect(i.Source, filepath.Dir(f.Path), getter.Detectors)
			if err != nil {
				resultErrLock.Lock()
				defer resultErrLock.Unlock()
				resultErr = multierror.Append(resultErr, fmt.Errorf(
					"获取import源错误: %s", err))
				return false
			}

			// Add this to the graph and check now if there are cycles
			graphLock.Lock()
			graph.Add(source)
			graph.Connect(dag.BasicEdge(parent, source))
			cycles := graph.Cycles()
			graphLock.Unlock()
			if len(cycles) > 0 {
				for _, cycle := range cycles {
					names := make([]string, len(cycle))
					for i, v := range cycle {
						names[i] = dag.VertexName(v)
					}

					resultErrLock.Lock()
					defer resultErrLock.Unlock()
					resultErr = multierror.Append(resultErr, fmt.Errorf(
						"Cycle found: %s", strings.Join(names, ", ")))
					return false
				}
			}
			wg.Add(1)
			go downloadSingle(source, &wg, &mergeLock, merge, idx)
		}

		// Wait for completion
		wg.Wait()

		// Go through the merge list and look for any nil entries, which
		// means that download failed. In that case, return immediately.
		// We assume any errors were put into resultErr.
		for _, importF := range merge {
			if importF == nil {
				return false
			}
		}

		for _, importF := range merge {
			// We need to copy importF here so that we don't poison
			// the cache by modifying the same pointer.
			importFCopy := *importF
			importF = &importFCopy
			source := importF.ID
			importF.ID = ""
			importF.Path = ""

			// Merge it into our file!
			if err := f.Merge(importF); err != nil {
				resultErrLock.Lock()
				defer resultErrLock.Unlock()
				resultErr = multierror.Append(resultErr, fmt.Errorf(
					"合并import错误 %s : %s", source, err))
				return false
			}
		}
		return true
	}

	// downloadSingle is used to download a single import and parse the
	// Appfile. This is a separate function because it is generally run
	// in a goroutine so we can parallelize grabbing the imports.
	downloadSingle = func(source string, wg *sync.WaitGroup, l *sync.Mutex, result []*File, idx int) {}

	importSingle("root", root)
	return resultErr
}
Ejemplo n.º 24
0
func (t *CreateBeforeDestroyTransformer) Transform(g *Graph) error {
	// We "stage" the edge connections/destroys in these slices so that
	// while we're doing the edge transformations (transpositions) in
	// the graph, we're not affecting future edge transpositions. These
	// slices let us stage ALL the changes that WILL happen so that all
	// of the transformations happen atomically.
	var connect, destroy []dag.Edge

	for _, v := range g.Vertices() {
		// We only care to use the destroy nodes
		dn, ok := v.(GraphNodeDestroy)
		if !ok {
			continue
		}

		// If the node doesn't need to create before destroy, then continue
		if !dn.CreateBeforeDestroy() {
			if noCreateBeforeDestroyAncestors(g, dn) {
				continue
			}

			// PURPOSELY HACKY FIX SINCE THIS TRANSFORM IS DEPRECATED.
			// This is a hacky way to fix GH-10439. For a detailed description
			// of the fix, see CBDEdgeTransformer, which is the equivalent
			// transform used by the new graphs.
			//
			// This transform is deprecated because it is only used by the
			// old graphs which are going to be removed.
			var update *config.Resource
			if dn, ok := v.(*graphNodeResourceDestroy); ok {
				update = dn.Original.Resource
			}
			if dn, ok := v.(*graphNodeResourceDestroyFlat); ok {
				update = dn.Original.Resource
			}
			if update != nil {
				update.Lifecycle.CreateBeforeDestroy = true
			}
		}

		// Get the creation side of this node
		cn := dn.CreateNode()

		// Take all the things which depend on the creation node and
		// make them dependencies on the destruction. Clarifying this
		// with an example: if you have a web server and a load balancer
		// and the load balancer depends on the web server, then when we
		// do a create before destroy, we want to make sure the steps are:
		//
		// 1.) Create new web server
		// 2.) Update load balancer
		// 3.) Delete old web server
		//
		// This ensures that.
		for _, sourceRaw := range g.UpEdges(cn).List() {
			source := sourceRaw.(dag.Vertex)

			// If the graph has a "root" node (one added by a RootTransformer and not
			// just a resource that happens to have no ancestors), we don't want to
			// add any edges to it, because then it ceases to be a root.
			if _, ok := source.(graphNodeRoot); ok {
				continue
			}

			connect = append(connect, dag.BasicEdge(dn, source))
		}

		// Swap the edge so that the destroy depends on the creation
		// happening...
		connect = append(connect, dag.BasicEdge(dn, cn))
		destroy = append(destroy, dag.BasicEdge(cn, dn))
	}

	for _, edge := range connect {
		g.Connect(edge)
	}
	for _, edge := range destroy {
		g.RemoveEdge(edge)
	}

	return nil
}
Ejemplo n.º 25
0
func (t *DestroyTransformer) Transform(g *Graph) error {
	var connect, remove []dag.Edge
	nodeToCn := make(map[dag.Vertex]dag.Vertex, len(g.Vertices()))
	nodeToDn := make(map[dag.Vertex]dag.Vertex, len(g.Vertices()))
	for _, v := range g.Vertices() {
		// If it is not a destroyable, we don't care
		cn, ok := v.(GraphNodeDestroyable)
		if !ok {
			continue
		}

		// Grab the destroy side of the node and connect it through
		n := cn.DestroyNode()
		if n == nil {
			continue
		}

		// Store it
		nodeToCn[n] = cn
		nodeToDn[cn] = n

		// If the creation node is equal to the destroy node, then
		// don't do any of the edge jump rope below.
		if n.(interface{}) == cn.(interface{}) {
			continue
		}

		// Add it to the graph
		g.Add(n)

		// Inherit all the edges from the old node
		downEdges := g.DownEdges(v).List()
		for _, edgeRaw := range downEdges {
			// If this thing specifically requests to not be depended on
			// by destroy nodes, then don't.
			if i, ok := edgeRaw.(GraphNodeDestroyEdgeInclude); ok &&
				!i.DestroyEdgeInclude(v) {
				continue
			}

			g.Connect(dag.BasicEdge(n, edgeRaw.(dag.Vertex)))
		}

		// Add a new edge to connect the node to be created to
		// the destroy node.
		connect = append(connect, dag.BasicEdge(v, n))
	}

	// Go through the nodes we added and determine if they depend
	// on any nodes with a destroy node. If so, depend on that instead.
	for n, _ := range nodeToCn {
		for _, downRaw := range g.DownEdges(n).List() {
			target := downRaw.(dag.Vertex)
			cn2, ok := target.(GraphNodeDestroyable)
			if !ok {
				continue
			}

			newTarget := nodeToDn[cn2]
			if newTarget == nil {
				continue
			}

			// Make the new edge and transpose
			connect = append(connect, dag.BasicEdge(newTarget, n))

			// Remove the old edge
			remove = append(remove, dag.BasicEdge(n, target))
		}
	}

	// Atomatically add/remove the edges
	for _, e := range connect {
		g.Connect(e)
	}
	for _, e := range remove {
		g.RemoveEdge(e)
	}

	return nil
}
Ejemplo n.º 26
0
func (t *DestroyTransformer) transform(
	g *Graph, mode GraphNodeDestroyMode) ([]dag.Edge, []dag.Edge, error) {
	var connect, remove []dag.Edge
	nodeToCn := make(map[dag.Vertex]dag.Vertex, len(g.Vertices()))
	nodeToDn := make(map[dag.Vertex]dag.Vertex, len(g.Vertices()))
	for _, v := range g.Vertices() {
		// If it is not a destroyable, we don't care
		cn, ok := v.(GraphNodeDestroyable)
		if !ok {
			continue
		}

		// Grab the destroy side of the node and connect it through
		n := cn.DestroyNode(mode)
		if n == nil {
			continue
		}

		// Store it
		nodeToCn[n] = cn
		nodeToDn[cn] = n

		// Add it to the graph
		g.Add(n)

		// Inherit all the edges from the old node
		downEdges := g.DownEdges(v).List()
		for _, edgeRaw := range downEdges {
			g.Connect(dag.BasicEdge(n, edgeRaw.(dag.Vertex)))
		}

		// Add a new edge to connect the node to be created to
		// the destroy node.
		connect = append(connect, dag.BasicEdge(v, n))
	}

	// Go through the nodes we added and determine if they depend
	// on any nodes with a destroy node. If so, depend on that instead.
	for n, _ := range nodeToCn {
		for _, downRaw := range g.DownEdges(n).List() {
			target := downRaw.(dag.Vertex)
			cn2, ok := target.(GraphNodeDestroyable)
			if !ok {
				continue
			}

			newTarget := nodeToDn[cn2]
			if newTarget == nil {
				continue
			}

			// Make the new edge and transpose
			connect = append(connect, dag.BasicEdge(newTarget, n))

			// Remove the old edge
			remove = append(remove, dag.BasicEdge(n, target))
		}
	}

	return connect, remove, nil
}
Ejemplo n.º 27
0
func (c *Compiler) compileDependencies(root *CompiledGraphVertex, graph *dag.AcyclicGraph) error {
	// For easier reference below
	storage := c.depStorage

	// Make a map to keep track of the dep source to vertex mapping
	vertexMap := make(map[string]*CompiledGraphVertex)

	// Store ourselves in the map
	key, err := getter.Detect(
		".", filepath.Dir(root.File.Path),
		getter.Detectors)
	if err != nil {
		return err
	}
	vertexMap[key] = root

	// Make a queue for the other vertices we need to still get
	// dependencies for. We arbitrarily make the cap for this slice
	// 30, since that is a ton of dependencies and we don't expect the
	// average case to have more than this.
	queue := make([]*CompiledGraphVertex, 1, 30)
	queue[0] = root

	// While we still have dependencies to get, continue loading them.
	// TODO: parallelize
	for len(queue) > 0 {
		var current *CompiledGraphVertex
		current, queue = queue[len(queue)-1], queue[:len(queue)-1]

		log.Printf("[DEBUG] compiling dependencies for: %s", current.Name())
		for _, dep := range current.File.Application.Dependencies {
			key, err := getter.Detect(
				dep.Source, filepath.Dir(current.File.Path),
				getter.Detectors)
			if err != nil {
				return fmt.Errorf(
					"Error loading source: %s", err)
			}

			vertex := vertexMap[key]
			if vertex == nil {
				log.Printf("[DEBUG] loading dependency: %s", key)

				// Call the callback if we have one
				if c.opts.Callback != nil {
					c.opts.Callback(&CompileEventDep{
						Source: key,
					})
				}

				// Download the dependency
				if err := storage.Get(key, key, true); err != nil {
					return err
				}
				dir, _, err := storage.Dir(key)
				if err != nil {
					return err
				}

				// Parse the Appfile if it exists
				var f *File
				appfilePath := filepath.Join(dir, "Appfile")
				_, err = os.Stat(appfilePath)
				if err != nil && !os.IsNotExist(err) {
					return fmt.Errorf(
						"Error parsing Appfile in %s: %s", key, err)
				}
				if err == nil {
					f, err = ParseFile(appfilePath)
					if err != nil {
						return fmt.Errorf(
							"Error parsing Appfile in %s: %s", key, err)
					}

					// Realize all the imports for this file
					if err := c.compileImports(f); err != nil {
						return err
					}
				}

				// Do any additional loading if we have a loader
				if c.opts.Loader != nil {
					f, err = c.opts.Loader(f, dir)
					if err != nil {
						return fmt.Errorf(
							"Error loading Appfile in %s: %s", key, err)
					}
				}

				// Set the source
				f.Source = key

				// If it doesn't have an otto ID then we can't do anything
				hasID, err := f.hasID()
				if err != nil {
					return fmt.Errorf(
						"Error checking for ID file for Appfile in %s: %s",
						key, err)
				}
				if !hasID {
					return fmt.Errorf(
						"Dependency '%s' doesn't have an Otto ID yet!\n\n"+
							"An Otto ID is generated on the first compilation of the Appfile.\n"+
							"It is a globally unique ID that is used to track the application\n"+
							"across multiple deploys. It is required for the application to be\n"+
							"used as a dependency. To fix this, check out that application and\n"+
							"compile the Appfile with `otto compile` once. Make sure you commit\n"+
							"the .ottoid file into version control, and then try this command\n"+
							"again.",
						key)
				}

				// We merge the root infrastructure choice upwards to
				// all dependencies.
				f.Infrastructure = root.File.Infrastructure
				if root.File.Project != nil {
					if f.Project == nil {
						f.Project = new(Project)
					}
					f.Project.Infrastructure = root.File.Project.Infrastructure
				}

				// Build the vertex for this
				vertex = &CompiledGraphVertex{
					File:      f,
					Dir:       dir,
					NameValue: f.Application.Name,
				}

				// Add the vertex since it is new, store the mapping, and
				// queue it to be loaded later.
				graph.Add(vertex)
				vertexMap[key] = vertex
				queue = append(queue, vertex)
			}

			// Connect the dependencies
			graph.Connect(dag.BasicEdge(current, vertex))
		}
	}

	return nil
}
func (t *CBDEdgeTransformer) Transform(g *Graph) error {
	log.Printf("[TRACE] CBDEdgeTransformer: Beginning CBD transformation...")

	// Go through and reverse any destroy edges
	destroyMap := make(map[string][]dag.Vertex)
	for _, v := range g.Vertices() {
		dn, ok := v.(GraphNodeDestroyerCBD)
		if !ok {
			continue
		}

		if !dn.CreateBeforeDestroy() {
			continue
		}

		// Find the destroy edge. There should only be one.
		for _, e := range g.EdgesTo(v) {
			// Not a destroy edge, ignore it
			de, ok := e.(*DestroyEdge)
			if !ok {
				continue
			}

			log.Printf("[TRACE] CBDEdgeTransformer: inverting edge: %s => %s",
				dag.VertexName(de.Source()), dag.VertexName(de.Target()))

			// Found it! Invert.
			g.RemoveEdge(de)
			g.Connect(&DestroyEdge{S: de.Target(), T: de.Source()})
		}

		// Add this to the list of nodes that we need to fix up
		// the edges for (step 2 above in the docs).
		key := dn.DestroyAddr().String()
		destroyMap[key] = append(destroyMap[key], v)
	}

	// If we have no CBD nodes, then our work here is done
	if len(destroyMap) == 0 {
		return nil
	}

	// We have CBD nodes. We now have to move on to the much more difficult
	// task of connecting dependencies of the creation side of the destroy
	// to the destruction node. The easiest way to explain this is an example:
	//
	// Given a pre-destroy dependence of: A => B
	//   And A has CBD set.
	//
	// The resulting graph should be: A => B => A_d
	//
	// They key here is that B happens before A is destroyed. This is to
	// facilitate the primary purpose for CBD: making sure that downstreams
	// are properly updated to avoid downtime before the resource is destroyed.
	//
	// We can't trust that the resource being destroyed or anything that
	// depends on it is actually in our current graph so we make a new
	// graph in order to determine those dependencies and add them in.
	log.Printf("[TRACE] CBDEdgeTransformer: building graph to find dependencies...")
	depMap, err := t.depMap(destroyMap)
	if err != nil {
		return err
	}

	// We now have the mapping of resource addresses to the destroy
	// nodes they need to depend on. We now go through our own vertices to
	// find any matching these addresses and make the connection.
	for _, v := range g.Vertices() {
		// We're looking for creators
		rn, ok := v.(GraphNodeCreator)
		if !ok {
			continue
		}

		// Get the address
		addr := rn.CreateAddr()
		key := addr.String()

		// If there is nothing this resource should depend on, ignore it
		dns, ok := depMap[key]
		if !ok {
			continue
		}

		// We have nodes! Make the connection
		for _, dn := range dns {
			log.Printf("[TRACE] CBDEdgeTransformer: destroy depends on dependence: %s => %s",
				dag.VertexName(dn), dag.VertexName(v))
			g.Connect(dag.BasicEdge(dn, v))
		}
	}

	return nil
}
Ejemplo n.º 29
0
func (l *Layered) graph(db *bolt.DB) (*dag.AcyclicGraph, error) {
	graph := new(dag.AcyclicGraph)
	graph.Add("root")

	// First, add all the layers
	layers := make(map[string]*layerVertex)
	err := db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(boltLayersBucket)
		return bucket.ForEach(func(k, data []byte) error {
			var v layerVertex
			if err := l.structRead(&v, data); err != nil {
				return err
			}

			// Add this layer to the graph
			graph.Add(&v)

			// Store the mapping for later
			layers[v.Layer.ID] = &v
			return nil
		})
	})
	if err != nil {
		return nil, err
	}

	// Next, connect the layers
	err = db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(boltEdgesBucket)
		return bucket.ForEach(func(k, data []byte) error {
			from := layers[string(k)]
			to := layers[string(data)]
			if from != nil && to != nil {
				graph.Connect(dag.BasicEdge(from, to))
			}

			return nil
		})
	})
	if err != nil {
		return nil, err
	}

	// Finally, add and connect all the envs
	err = db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(boltEnvsBucket)
		return bucket.ForEach(func(k, data []byte) error {
			key := fmt.Sprintf("env-%s", string(k))
			graph.Add(key)

			// Connect the env to the layer it depends on
			to := &layerVertex{Layer: &Layer{ID: string(data)}}
			graph.Connect(dag.BasicEdge(key, to))

			// Connect the root to the environment that is active
			graph.Connect(dag.BasicEdge("root", key))
			return nil
		})
	})
	if err != nil {
		return nil, err
	}

	return graph, nil
}
Ejemplo n.º 30
0
// compileImports takes a File, loads all the imports, and merges them
// into the File.
func (c *Compiler) compileImports(root *File) error {
	// If we have no imports, short-circuit the whole thing
	if len(root.Imports) == 0 {
		return nil
	}

	// Pull these out into variables so they're easier to reference
	storage := c.importStorage
	cache := c.importCache
	cacheLock := &c.importLock

	// A graph is used to track for cycles
	var graphLock sync.Mutex
	graph := new(dag.AcyclicGraph)
	graph.Add("root")

	// Since we run the import in parallel, multiple errors can happen
	// at the same time. We use multierror and a lock to keep track of errors.
	var resultErr error
	var resultErrLock sync.Mutex

	// Forward declarations for some nested functions we use. The docs
	// for these functions are above each.
	var importSingle func(parent string, f *File) bool
	var downloadSingle func(string, *sync.WaitGroup, *sync.Mutex, []*File, int)

	// importSingle is responsible for kicking off the imports and merging
	// them for a single file. This will return true on success, false on
	// failure. On failure, it is expected that any errors are appended to
	// resultErr.
	importSingle = func(parent string, f *File) bool {
		var wg sync.WaitGroup

		// Build the list of files we'll merge later
		var mergeLock sync.Mutex
		merge := make([]*File, len(f.Imports))

		// Go through the imports and kick off the download
		for idx, i := range f.Imports {
			source, err := getter.Detect(
				i.Source, filepath.Dir(f.Path),
				getter.Detectors)
			if err != nil {
				resultErrLock.Lock()
				defer resultErrLock.Unlock()
				resultErr = multierror.Append(resultErr, fmt.Errorf(
					"Error loading import source: %s", err))
				return false
			}

			// Add this to the graph and check now if there are cycles
			graphLock.Lock()
			graph.Add(source)
			graph.Connect(dag.BasicEdge(parent, source))
			cycles := graph.Cycles()
			graphLock.Unlock()
			if len(cycles) > 0 {
				for _, cycle := range cycles {
					names := make([]string, len(cycle))
					for i, v := range cycle {
						names[i] = dag.VertexName(v)
					}

					resultErrLock.Lock()
					defer resultErrLock.Unlock()
					resultErr = multierror.Append(resultErr, fmt.Errorf(
						"Cycle found: %s", strings.Join(names, ", ")))
					return false
				}
			}

			wg.Add(1)
			go downloadSingle(source, &wg, &mergeLock, merge, idx)
		}

		// Wait for completion
		wg.Wait()

		// Go through the merge list and look for any nil entries, which
		// means that download failed. In that case, return immediately.
		// We assume any errors were put into resultErr.
		for _, importF := range merge {
			if importF == nil {
				return false
			}
		}

		for _, importF := range merge {
			// We need to copy importF here so that we don't poison
			// the cache by modifying the same pointer.
			importFCopy := *importF
			importF = &importFCopy
			source := importF.ID
			importF.ID = ""
			importF.Path = ""

			// Merge it into our file!
			if err := f.Merge(importF); err != nil {
				resultErrLock.Lock()
				defer resultErrLock.Unlock()
				resultErr = multierror.Append(resultErr, fmt.Errorf(
					"Error merging import %s: %s", source, err))
				return false
			}
		}

		return true
	}

	// downloadSingle is used to download a single import and parse the
	// Appfile. This is a separate function because it is generally run
	// in a goroutine so we can parallelize grabbing the imports.
	downloadSingle = func(source string, wg *sync.WaitGroup, l *sync.Mutex, result []*File, idx int) {
		defer wg.Done()

		// Read from the cache if we have it
		cacheLock.Lock()
		cached, ok := cache[source]
		cacheLock.Unlock()
		if ok {
			log.Printf("[DEBUG] cache hit on import: %s", source)
			l.Lock()
			defer l.Unlock()
			result[idx] = cached
			return
		}

		// Call the callback if we have one
		log.Printf("[DEBUG] loading import: %s", source)
		if c.opts.Callback != nil {
			c.opts.Callback(&CompileEventImport{
				Source: source,
			})
		}

		// Download the dependency
		if err := storage.Get(source, source, true); err != nil {
			resultErrLock.Lock()
			defer resultErrLock.Unlock()
			resultErr = multierror.Append(resultErr, fmt.Errorf(
				"Error loading import source: %s", err))
			return
		}
		dir, _, err := storage.Dir(source)
		if err != nil {
			resultErrLock.Lock()
			defer resultErrLock.Unlock()
			resultErr = multierror.Append(resultErr, fmt.Errorf(
				"Error loading import source: %s", err))
			return
		}

		// Parse the Appfile
		importF, err := ParseFile(filepath.Join(dir, "Appfile"))
		if err != nil {
			resultErrLock.Lock()
			defer resultErrLock.Unlock()
			resultErr = multierror.Append(resultErr, fmt.Errorf(
				"Error parsing Appfile in %s: %s", source, err))
			return
		}

		// We use the ID to store the source, but we clear it
		// when we actually merge.
		importF.ID = source

		// Import the imports in this
		if !importSingle(source, importF) {
			return
		}

		// Once we're done, acquire the lock and write it
		l.Lock()
		result[idx] = importF
		l.Unlock()

		// Write this into the cache.
		cacheLock.Lock()
		cache[source] = importF
		cacheLock.Unlock()
	}

	importSingle("root", root)
	return resultErr
}