Beispiel #1
0
// NewGraphFromConfig transforms a GraphConfig struct into a new graph.
// FIXME: remove any possibly left over, now obsolete graph diff code from here!
func (c *GraphConfig) NewGraphFromConfig(hostname string, world gapi.World, noop bool) (*pgraph.Graph, error) {
	// hostname is the uuid for the host

	var graph *pgraph.Graph          // new graph to return
	graph = pgraph.NewGraph("Graph") // give graph a default name

	var lookup = make(map[string]map[string]*pgraph.Vertex)

	//log.Printf("%+v", config) // debug

	// TODO: if defined (somehow)...
	graph.SetName(c.Graph) // set graph name

	var keep []*pgraph.Vertex        // list of vertex which are the same in new graph
	var resourceList []resources.Res // list of resources to export
	// use reflection to avoid duplicating code... better options welcome!
	value := reflect.Indirect(reflect.ValueOf(c.Resources))
	vtype := value.Type()
	for i := 0; i < vtype.NumField(); i++ { // number of fields in struct
		name := vtype.Field(i).Name // string of field name
		field := value.FieldByName(name)
		iface := field.Interface() // interface type of value
		slice := reflect.ValueOf(iface)
		// XXX: should we just drop these everywhere and have the kind strings be all lowercase?
		kind := util.FirstToUpper(name)
		if Debug {
			log.Printf("Config: Processing: %v...", kind)
		}
		for j := 0; j < slice.Len(); j++ { // loop through resources of same kind
			x := slice.Index(j).Interface()
			res, ok := x.(resources.Res) // convert to Res type
			if !ok {
				return nil, fmt.Errorf("Config: Error: Can't convert: %v of type: %T to Res.", x, x)
			}
			//if noop { // now done in mgmtmain
			//	res.Meta().Noop = noop
			//}
			if _, exists := lookup[kind]; !exists {
				lookup[kind] = make(map[string]*pgraph.Vertex)
			}
			// XXX: should we export based on a @@ prefix, or a metaparam
			// like exported => true || exported => (host pattern)||(other pattern?)
			if !strings.HasPrefix(res.GetName(), "@@") { // not exported resource
				v := graph.GetVertexMatch(res)
				if v == nil { // no match found
					res.Init()
					v = pgraph.NewVertex(res)
					graph.AddVertex(v) // call standalone in case not part of an edge
				}
				lookup[kind][res.GetName()] = v // used for constructing edges
				keep = append(keep, v)          // append

			} else if !noop { // do not export any resources if noop
				// store for addition to backend storage...
				res.SetName(res.GetName()[2:]) //slice off @@
				res.SetKind(kind)              // cheap init
				resourceList = append(resourceList, res)
			}
		}
	}
	// store in backend (usually etcd)
	if err := world.ResExport(resourceList); err != nil {
		return nil, fmt.Errorf("Config: Could not export resources: %v", err)
	}

	// lookup from backend (usually etcd)
	var hostnameFilter []string // empty to get from everyone
	kindFilter := []string{}
	for _, t := range c.Collector {
		// XXX: should we just drop these everywhere and have the kind strings be all lowercase?
		kind := util.FirstToUpper(t.Kind)
		kindFilter = append(kindFilter, kind)
	}
	// do all the graph look ups in one single step, so that if the backend
	// database changes, we don't have a partial state of affairs...
	if len(kindFilter) > 0 { // if kindFilter is empty, don't need to do lookups!
		var err error
		resourceList, err = world.ResCollect(hostnameFilter, kindFilter)
		if err != nil {
			return nil, fmt.Errorf("Config: Could not collect resources: %v", err)
		}
	}
	for _, res := range resourceList {
		matched := false
		// see if we find a collect pattern that matches
		for _, t := range c.Collector {
			// XXX: should we just drop these everywhere and have the kind strings be all lowercase?
			kind := util.FirstToUpper(t.Kind)
			// use t.Kind and optionally t.Pattern to collect from storage
			log.Printf("Collect: %v; Pattern: %v", kind, t.Pattern)

			// XXX: expand to more complex pattern matching here...
			if res.Kind() != kind {
				continue
			}

			if matched {
				// we've already matched this resource, should we match again?
				log.Printf("Config: Warning: Matching %v[%v] again!", kind, res.GetName())
			}
			matched = true

			// collect resources but add the noop metaparam
			//if noop { // now done in mgmtmain
			//	res.Meta().Noop = noop
			//}

			if t.Pattern != "" { // XXX: simplistic for now
				res.CollectPattern(t.Pattern) // res.Dirname = t.Pattern
			}

			log.Printf("Collect: %v[%v]: collected!", kind, res.GetName())

			// XXX: similar to other resource add code:
			if _, exists := lookup[kind]; !exists {
				lookup[kind] = make(map[string]*pgraph.Vertex)
			}
			v := graph.GetVertexMatch(res)
			if v == nil { // no match found
				res.Init() // initialize go channels or things won't work!!!
				v = pgraph.NewVertex(res)
				graph.AddVertex(v) // call standalone in case not part of an edge
			}
			lookup[kind][res.GetName()] = v // used for constructing edges
			keep = append(keep, v)          // append

			//break // let's see if another resource even matches
		}
	}

	for _, e := range c.Edges {
		if _, ok := lookup[util.FirstToUpper(e.From.Kind)]; !ok {
			return nil, fmt.Errorf("Can't find 'from' resource!")
		}
		if _, ok := lookup[util.FirstToUpper(e.To.Kind)]; !ok {
			return nil, fmt.Errorf("Can't find 'to' resource!")
		}
		if _, ok := lookup[util.FirstToUpper(e.From.Kind)][e.From.Name]; !ok {
			return nil, fmt.Errorf("Can't find 'from' name!")
		}
		if _, ok := lookup[util.FirstToUpper(e.To.Kind)][e.To.Name]; !ok {
			return nil, fmt.Errorf("Can't find 'to' name!")
		}
		graph.AddEdge(lookup[util.FirstToUpper(e.From.Kind)][e.From.Name], lookup[util.FirstToUpper(e.To.Kind)][e.To.Name], pgraph.NewEdge(e.Name))
	}

	return graph, nil
}
Beispiel #2
0
// Run is the main execution entrypoint to run mgmt.
func (obj *Main) Run() error {

	var start = time.Now().UnixNano()

	var flags int
	if obj.DEBUG || true { // TODO: remove || true
		flags = log.LstdFlags | log.Lshortfile
	}
	flags = (flags - log.Ldate) // remove the date for now
	log.SetFlags(flags)

	// un-hijack from capnslog...
	log.SetOutput(os.Stderr)
	if obj.VERBOSE {
		capnslog.SetFormatter(capnslog.NewLogFormatter(os.Stderr, "(etcd) ", flags))
	} else {
		capnslog.SetFormatter(capnslog.NewNilFormatter())
	}

	log.Printf("This is: %s, version: %s", obj.Program, obj.Version)
	log.Printf("Main: Start: %v", start)

	hostname, err := os.Hostname() // a sensible default
	// allow passing in the hostname, instead of using the system setting
	if h := obj.Hostname; h != nil && *h != "" { // override by cli
		hostname = *h
	} else if err != nil {
		return errwrap.Wrapf(err, "Can't get default hostname!")
	}
	if hostname == "" { // safety check
		return fmt.Errorf("Hostname cannot be empty!")
	}

	var prefix = fmt.Sprintf("/var/lib/%s/", obj.Program) // default prefix
	if p := obj.Prefix; p != nil {
		prefix = *p
	}
	// make sure the working directory prefix exists
	if obj.TmpPrefix || os.MkdirAll(prefix, 0770) != nil {
		if obj.TmpPrefix || obj.AllowTmpPrefix {
			var err error
			if prefix, err = ioutil.TempDir("", obj.Program+"-"+hostname+"-"); err != nil {
				return fmt.Errorf("Main: Error: Can't create temporary prefix!")
			}
			log.Println("Main: Warning: Working prefix directory is temporary!")

		} else {
			return fmt.Errorf("Main: Error: Can't create prefix!")
		}
	}
	log.Printf("Main: Working prefix is: %s", prefix)

	var wg sync.WaitGroup
	var G, oldGraph *pgraph.Graph

	// exit after `max-runtime` seconds for no reason at all...
	if i := obj.MaxRuntime; i > 0 {
		go func() {
			time.Sleep(time.Duration(i) * time.Second)
			obj.Exit(nil)
		}()
	}

	// setup converger
	converger := converger.NewConverger(
		obj.ConvergedTimeout,
		nil, // stateFn gets added in by EmbdEtcd
	)
	go converger.Loop(true) // main loop for converger, true to start paused

	// embedded etcd
	if len(obj.seeds) == 0 {
		log.Printf("Main: Seeds: No seeds specified!")
	} else {
		log.Printf("Main: Seeds(%d): %v", len(obj.seeds), obj.seeds)
	}
	EmbdEtcd := etcd.NewEmbdEtcd(
		hostname,
		obj.seeds,
		obj.clientURLs,
		obj.serverURLs,
		obj.NoServer,
		obj.idealClusterSize,
		prefix,
		converger,
	)
	if EmbdEtcd == nil {
		// TODO: verify EmbdEtcd is not nil below...
		obj.Exit(fmt.Errorf("Main: Etcd: Creation failed!"))
	} else if err := EmbdEtcd.Startup(); err != nil { // startup (returns when etcd main loop is running)
		obj.Exit(fmt.Errorf("Main: Etcd: Startup failed: %v", err))
	}
	convergerStateFn := func(b bool) error {
		// exit if we are using the converged timeout and we are the
		// root node. otherwise, if we are a child node in a remote
		// execution hierarchy, we should only notify our converged
		// state and wait for the parent to trigger the exit.
		if t := obj.ConvergedTimeout; obj.Depth == 0 && t >= 0 {
			if b {
				log.Printf("Converged for %d seconds, exiting!", t)
				obj.Exit(nil) // trigger an exit!
			}
			return nil
		}
		// send our individual state into etcd for others to see
		return etcd.EtcdSetHostnameConverged(EmbdEtcd, hostname, b) // TODO: what should happen on error?
	}
	if EmbdEtcd != nil {
		converger.SetStateFn(convergerStateFn)
	}

	var gapiChan chan error // stream events are nil errors
	if obj.GAPI != nil {
		data := gapi.Data{
			Hostname: hostname,
			EmbdEtcd: EmbdEtcd,
			Noop:     obj.Noop,
			NoWatch:  obj.NoWatch,
		}
		if err := obj.GAPI.Init(data); err != nil {
			obj.Exit(fmt.Errorf("Main: GAPI: Init failed: %v", err))
		} else if !obj.NoWatch {
			gapiChan = obj.GAPI.SwitchStream() // stream of graph switch events!
		}
	}

	exitchan := make(chan struct{}) // exit on close
	go func() {
		startChan := make(chan struct{}) // start signal
		go func() { startChan <- struct{}{} }()

		log.Println("Etcd: Starting...")
		etcdChan := etcd.EtcdWatch(EmbdEtcd)
		first := true // first loop or not
		for {
			log.Println("Main: Waiting...")
			select {
			case <-startChan: // kick the loop once at start
				// pass

			case b := <-etcdChan:
				if !b { // ignore the message
					continue
				}
				// everything else passes through to cause a compile!

			case err, ok := <-gapiChan:
				if !ok { // channel closed
					if obj.DEBUG {
						log.Printf("Main: GAPI exited")
					}
					gapiChan = nil // disable it
					continue
				}
				if err != nil {
					obj.Exit(err) // trigger exit
					continue
					//return // TODO: return or wait for exitchan?
				}
				if obj.NoWatch { // extra safety for bad GAPI's
					log.Printf("Main: GAPI stream should be quiet with NoWatch!") // fix the GAPI!
					continue                                                      // no stream events should be sent
				}

			case <-exitchan:
				return
			}

			if obj.GAPI == nil {
				log.Printf("Config: GAPI is empty!")
				continue
			}

			// we need the vertices to be paused to work on them, so
			// run graph vertex LOCK...
			if !first { // TODO: we can flatten this check out I think
				converger.Pause() // FIXME: add sync wait?
				G.Pause()         // sync

				//G.UnGroup() // FIXME: implement me if needed!
			}

			// make the graph from yaml, lib, puppet->yaml, or dsl!
			newGraph, err := obj.GAPI.Graph() // generate graph!
			if err != nil {
				log.Printf("Config: Error creating new graph: %v", err)
				// unpause!
				if !first {
					G.Start(&wg, first) // sync
					converger.Start()   // after G.Start()
				}
				continue
			}

			// apply the global noop parameter if requested
			if obj.Noop {
				for _, m := range newGraph.GraphMetas() {
					m.Noop = obj.Noop
				}
			}

			// FIXME: make sure we "UnGroup()" any semi-destructive
			// changes to the resources so our efficient GraphSync
			// will be able to re-use and cmp to the old graph.
			newFullGraph, err := newGraph.GraphSync(oldGraph)
			if err != nil {
				log.Printf("Config: Error running graph sync: %v", err)
				// unpause!
				if !first {
					G.Start(&wg, first) // sync
					converger.Start()   // after G.Start()
				}
				continue
			}
			oldGraph = newFullGraph // save old graph
			G = oldGraph.Copy()     // copy to active graph

			G.AutoEdges() // add autoedges; modifies the graph
			G.AutoGroup() // run autogroup; modifies the graph
			// TODO: do we want to do a transitive reduction?

			log.Printf("Graph: %v", G) // show graph
			if obj.GraphvizFilter != "" {
				if err := G.ExecGraphviz(obj.GraphvizFilter, obj.Graphviz); err != nil {
					log.Printf("Graphviz: %v", err)
				} else {
					log.Printf("Graphviz: Successfully generated graph!")
				}
			}
			G.AssociateData(converger)
			// G.Start(...) needs to be synchronous or wait,
			// because if half of the nodes are started and
			// some are not ready yet and the EtcdWatch
			// loops, we'll cause G.Pause(...) before we
			// even got going, thus causing nil pointer errors
			G.Start(&wg, first) // sync
			converger.Start()   // after G.Start()
			first = false
		}
	}()

	configWatcher := recwatch.NewConfigWatcher()
	events := configWatcher.Events()
	if !obj.NoWatch {
		configWatcher.Add(obj.Remotes...) // add all the files...
	} else {
		events = nil // signal that no-watch is true
	}
	go func() {
		select {
		case err := <-configWatcher.Error():
			obj.Exit(err) // trigger an exit!

		case <-exitchan:
			return
		}
	}()

	// initialize the add watcher, which calls the f callback on map changes
	convergerCb := func(f func(map[string]bool) error) (func(), error) {
		return etcd.EtcdAddHostnameConvergedWatcher(EmbdEtcd, f)
	}

	// build remotes struct for remote ssh
	remotes := remote.NewRemotes(
		EmbdEtcd.LocalhostClientURLs().StringSlice(),
		[]string{etcd.DefaultClientURL},
		obj.Noop,
		obj.Remotes, // list of files
		events,      // watch for file changes
		obj.CConns,
		obj.AllowInteractive,
		obj.SSHPrivIDRsa,
		!obj.NoCaching,
		obj.Depth,
		prefix,
		converger,
		convergerCb,
		obj.Program,
	)

	// TODO: is there any benefit to running the remotes above in the loop?
	// wait for etcd to be running before we remote in, which we do above!
	go remotes.Run()

	if obj.GAPI == nil {
		converger.Start() // better start this for empty graphs
	}
	log.Println("Main: Running...")

	reterr := <-obj.exit // wait for exit signal

	log.Println("Destroy...")

	if obj.GAPI != nil {
		if err := obj.GAPI.Close(); err != nil {
			err = errwrap.Wrapf(err, "GAPI closed poorly!")
			reterr = multierr.Append(reterr, err) // list of errors
		}
	}

	configWatcher.Close()                  // stop sending file changes to remotes
	if err := remotes.Exit(); err != nil { // tell all the remote connections to shutdown; waits!
		err = errwrap.Wrapf(err, "Remote exited poorly!")
		reterr = multierr.Append(reterr, err) // list of errors
	}

	G.Exit() // tell all the children to exit

	// tell inner main loop to exit
	close(exitchan)

	// cleanup etcd main loop last so it can process everything first
	if err := EmbdEtcd.Destroy(); err != nil { // shutdown and cleanup etcd
		err = errwrap.Wrapf(err, "Etcd exited poorly!")
		reterr = multierr.Append(reterr, err) // list of errors
	}

	if obj.DEBUG {
		log.Printf("Main: Graph: %v", G)
	}

	wg.Wait() // wait for primary go routines to exit

	// TODO: wait for each vertex to exit...
	log.Println("Goodbye!")
	return reterr
}
Beispiel #3
0
// run is the main run target.
func run(c *cli.Context) error {
	var start = time.Now().UnixNano()
	log.Printf("This is: %v, version: %v", program, version)
	log.Printf("Main: Start: %v", start)

	hostname, _ := os.Hostname()
	// allow passing in the hostname, instead of using --hostname
	if c.IsSet("file") {
		if config := gconfig.ParseConfigFromFile(c.String("file")); config != nil {
			if h := config.Hostname; h != "" {
				hostname = h
			}
		}
	}
	if c.IsSet("hostname") { // override by cli
		if h := c.String("hostname"); h != "" {
			hostname = h
		}
	}
	noop := c.Bool("noop")

	seeds, err := etcdtypes.NewURLs(
		util.FlattenListWithSplit(c.StringSlice("seeds"), []string{",", ";", " "}),
	)
	if err != nil && len(c.StringSlice("seeds")) > 0 {
		log.Printf("Main: Error: seeds didn't parse correctly!")
		return cli.NewExitError("", 1)
	}
	clientURLs, err := etcdtypes.NewURLs(
		util.FlattenListWithSplit(c.StringSlice("client-urls"), []string{",", ";", " "}),
	)
	if err != nil && len(c.StringSlice("client-urls")) > 0 {
		log.Printf("Main: Error: clientURLs didn't parse correctly!")
		return cli.NewExitError("", 1)
	}
	serverURLs, err := etcdtypes.NewURLs(
		util.FlattenListWithSplit(c.StringSlice("server-urls"), []string{",", ";", " "}),
	)
	if err != nil && len(c.StringSlice("server-urls")) > 0 {
		log.Printf("Main: Error: serverURLs didn't parse correctly!")
		return cli.NewExitError("", 1)
	}

	idealClusterSize := uint16(c.Int("ideal-cluster-size"))
	if idealClusterSize < 1 {
		log.Printf("Main: Error: idealClusterSize should be at least one!")
		return cli.NewExitError("", 1)
	}

	if c.IsSet("file") && c.IsSet("puppet") {
		log.Println("Main: Error: the --file and --puppet parameters cannot be used together!")
		return cli.NewExitError("", 1)
	}

	if c.Bool("no-server") && len(c.StringSlice("remote")) > 0 {
		// TODO: in this case, we won't be able to tunnel stuff back to
		// here, so if we're okay with every remote graph running in an
		// isolated mode, then this is okay. Improve on this if there's
		// someone who really wants to be able to do this.
		log.Println("Main: Error: the --no-server and --remote parameters cannot be used together!")
		return cli.NewExitError("", 1)
	}

	cConns := uint16(c.Int("cconns"))
	if cConns < 0 {
		log.Printf("Main: Error: --cconns should be at least zero!")
		return cli.NewExitError("", 1)
	}

	if c.IsSet("converged-timeout") && cConns > 0 && len(c.StringSlice("remote")) > c.Int("cconns") {
		log.Printf("Main: Error: combining --converged-timeout with more remotes than available connections will never converge!")
		return cli.NewExitError("", 1)
	}

	depth := uint16(c.Int("depth"))
	if depth < 0 { // user should not be using this argument manually
		log.Printf("Main: Error: negative values for --depth are not permitted!")
		return cli.NewExitError("", 1)
	}

	if c.IsSet("prefix") && c.Bool("tmp-prefix") {
		log.Println("Main: Error: combining --prefix and the request for a tmp prefix is illogical!")
		return cli.NewExitError("", 1)
	}
	if s := c.String("prefix"); c.IsSet("prefix") && s != "" {
		prefix = s
	}

	// make sure the working directory prefix exists
	if c.Bool("tmp-prefix") || os.MkdirAll(prefix, 0770) != nil {
		if c.Bool("tmp-prefix") || c.Bool("allow-tmp-prefix") {
			if prefix, err = ioutil.TempDir("", program+"-"); err != nil {
				log.Printf("Main: Error: Can't create temporary prefix!")
				return cli.NewExitError("", 1)
			}
			log.Println("Main: Warning: Working prefix directory is temporary!")

		} else {
			log.Printf("Main: Error: Can't create prefix!")
			return cli.NewExitError("", 1)
		}
	}
	log.Printf("Main: Working prefix is: %s", prefix)

	var wg sync.WaitGroup
	exit := make(chan bool) // exit signal
	var G, fullGraph *pgraph.Graph

	// exit after `max-runtime` seconds for no reason at all...
	if i := c.Int("max-runtime"); i > 0 {
		go func() {
			time.Sleep(time.Duration(i) * time.Second)
			exit <- true
		}()
	}

	// setup converger
	converger := converger.NewConverger(
		c.Int("converged-timeout"),
		nil, // stateFn gets added in by EmbdEtcd
	)
	go converger.Loop(true) // main loop for converger, true to start paused

	// embedded etcd
	if len(seeds) == 0 {
		log.Printf("Main: Seeds: No seeds specified!")
	} else {
		log.Printf("Main: Seeds(%v): %v", len(seeds), seeds)
	}
	EmbdEtcd := etcd.NewEmbdEtcd(
		hostname,
		seeds,
		clientURLs,
		serverURLs,
		c.Bool("no-server"),
		idealClusterSize,
		prefix,
		converger,
	)
	if EmbdEtcd == nil {
		// TODO: verify EmbdEtcd is not nil below...
		log.Printf("Main: Etcd: Creation failed!")
		exit <- true
	} else if err := EmbdEtcd.Startup(); err != nil { // startup (returns when etcd main loop is running)
		log.Printf("Main: Etcd: Startup failed: %v", err)
		exit <- true
	}
	convergerStateFn := func(b bool) error {
		// exit if we are using the converged-timeout and we are the
		// root node. otherwise, if we are a child node in a remote
		// execution hierarchy, we should only notify our converged
		// state and wait for the parent to trigger the exit.
		if depth == 0 && c.Int("converged-timeout") >= 0 {
			if b {
				log.Printf("Converged for %d seconds, exiting!", c.Int("converged-timeout"))
				exit <- true // trigger an exit!
			}
			return nil
		}
		// send our individual state into etcd for others to see
		return etcd.EtcdSetHostnameConverged(EmbdEtcd, hostname, b) // TODO: what should happen on error?
	}
	if EmbdEtcd != nil {
		converger.SetStateFn(convergerStateFn)
	}

	exitchan := make(chan struct{}) // exit on close
	go func() {
		startchan := make(chan struct{}) // start signal
		go func() { startchan <- struct{}{} }()
		file := c.String("file")
		var configchan chan bool
		var puppetchan <-chan time.Time
		if !c.Bool("no-watch") && c.IsSet("file") {
			configchan = ConfigWatch(file)
		} else if c.IsSet("puppet") {
			interval := puppet.PuppetInterval(c.String("puppet-conf"))
			puppetchan = time.Tick(time.Duration(interval) * time.Second)
		}
		log.Println("Etcd: Starting...")
		etcdchan := etcd.EtcdWatch(EmbdEtcd)
		first := true // first loop or not
		for {
			log.Println("Main: Waiting...")
			select {
			case <-startchan: // kick the loop once at start
				// pass

			case b := <-etcdchan:
				if !b { // ignore the message
					continue
				}
				// everything else passes through to cause a compile!

			case <-puppetchan:
				// nothing, just go on

			case msg := <-configchan:
				if c.Bool("no-watch") || !msg {
					continue // not ready to read config
				}
			// XXX: case compile_event: ...
			// ...
			case <-exitchan:
				return
			}

			var config *gconfig.GraphConfig
			if c.IsSet("file") {
				config = gconfig.ParseConfigFromFile(file)
			} else if c.IsSet("puppet") {
				config = puppet.ParseConfigFromPuppet(c.String("puppet"), c.String("puppet-conf"))
			}
			if config == nil {
				log.Printf("Config: Parse failure")
				continue
			}

			if config.Hostname != "" && config.Hostname != hostname {
				log.Printf("Config: Hostname changed, ignoring config!")
				continue
			}
			config.Hostname = hostname // set it in case it was ""

			// run graph vertex LOCK...
			if !first { // TODO: we can flatten this check out I think
				converger.Pause() // FIXME: add sync wait?
				G.Pause()         // sync
			}

			// build graph from yaml file on events (eg: from etcd)
			// we need the vertices to be paused to work on them
			if newFullgraph, err := config.NewGraphFromConfig(fullGraph, EmbdEtcd, noop); err == nil { // keep references to all original elements
				fullGraph = newFullgraph
			} else {
				log.Printf("Config: Error making new graph from config: %v", err)
				// unpause!
				if !first {
					G.Start(&wg, first) // sync
					converger.Start()   // after G.Start()
				}
				continue
			}

			G = fullGraph.Copy() // copy to active graph
			// XXX: do etcd transaction out here...
			G.AutoEdges() // add autoedges; modifies the graph
			G.AutoGroup() // run autogroup; modifies the graph
			// TODO: do we want to do a transitive reduction?

			log.Printf("Graph: %v", G) // show graph
			err := G.ExecGraphviz(c.String("graphviz-filter"), c.String("graphviz"))
			if err != nil {
				log.Printf("Graphviz: %v", err)
			} else {
				log.Printf("Graphviz: Successfully generated graph!")
			}
			G.AssociateData(converger)
			// G.Start(...) needs to be synchronous or wait,
			// because if half of the nodes are started and
			// some are not ready yet and the EtcdWatch
			// loops, we'll cause G.Pause(...) before we
			// even got going, thus causing nil pointer errors
			G.Start(&wg, first) // sync
			converger.Start()   // after G.Start()
			first = false
		}
	}()

	configWatcher := NewConfigWatcher()
	events := configWatcher.Events()
	if !c.Bool("no-watch") {
		configWatcher.Add(c.StringSlice("remote")...) // add all the files...
	} else {
		events = nil // signal that no-watch is true
	}

	// initialize the add watcher, which calls the f callback on map changes
	convergerCb := func(f func(map[string]bool) error) (func(), error) {
		return etcd.EtcdAddHostnameConvergedWatcher(EmbdEtcd, f)
	}

	// build remotes struct for remote ssh
	remotes := remote.NewRemotes(
		EmbdEtcd.LocalhostClientURLs().StringSlice(),
		[]string{etcd.DefaultClientURL},
		noop,
		c.StringSlice("remote"), // list of files
		events,                  // watch for file changes
		cConns,
		c.Bool("allow-interactive"),
		c.String("ssh-priv-id-rsa"),
		!c.Bool("no-caching"),
		depth,
		prefix,
		converger,
		convergerCb,
		program,
	)

	// TODO: is there any benefit to running the remotes above in the loop?
	// wait for etcd to be running before we remote in, which we do above!
	go remotes.Run()

	if !c.IsSet("file") && !c.IsSet("puppet") {
		converger.Start() // better start this for empty graphs
	}
	log.Println("Main: Running...")

	waitForSignal(exit) // pass in exit channel to watch

	log.Println("Destroy...")

	configWatcher.Close() // stop sending file changes to remotes
	remotes.Exit()        // tell all the remote connections to shutdown; waits!

	G.Exit() // tell all the children to exit

	// tell inner main loop to exit
	close(exitchan)

	// cleanup etcd main loop last so it can process everything first
	if err := EmbdEtcd.Destroy(); err != nil { // shutdown and cleanup etcd
		log.Printf("Etcd exited poorly with: %v", err)
	}

	if global.DEBUG {
		log.Printf("Graph: %v", G)
	}

	wg.Wait() // wait for primary go routines to exit

	// TODO: wait for each vertex to exit...
	log.Println("Goodbye!")
	return nil
}