Beispiel #1
0
// deployDiscoverMeta does a one-by-one deployment but uses discoverd.Deployment
// to wait for appropriate service metadata before stopping old jobs.
func (d *DeployJob) deployDiscoverdMeta() (err error) {
	log := d.logger.New("fn", "deployDiscoverdMeta")
	log.Info("starting discoverd-meta deployment")

	defer func() {
		if err != nil {
			// TODO: support rolling back
			err = ErrSkipRollback{err.Error()}
		}
	}()

	discDeploys := make(map[string]*dd.Deployment)

	for typ, serviceName := range d.serviceNames {
		discDeploy, err := dd.NewDeployment(serviceName)
		if err != nil {
			return err
		}
		discDeploys[typ] = discDeploy
		if err := discDeploy.Create(d.ID); err != nil {
			return err
		}
		defer discDeploy.Close()
	}

	return d.deployOneByOneWithWaitFn(func(releaseID string, expected ct.JobEvents, log log15.Logger) error {
		for typ, events := range expected {
			if count, ok := events[ct.JobStateUp]; ok && count > 0 {
				if discDeploy, ok := discDeploys[typ]; ok {
					if err := discDeploy.Wait(d.ID, count, 120, log); err != nil {
						return err
					}
					// clear up events for this type so we can safely
					// process job down events if needed
					expected[typ][ct.JobStateUp] = 0
				}
			}
		}
		if expected.Count() == 0 {
			return nil
		}
		return d.waitForJobEvents(releaseID, expected, log)
	})
}
Beispiel #2
0
// Run executes the program.
func (m *Main) Run(args ...string) error {
	// Create logger.
	m.logger = log.New(m.Stdout, "", log.LstdFlags)

	// Parse command line flags.
	opt, err := m.ParseFlags(args...)
	if err != nil {
		return err
	}

	// Set up advertised address and default peer set.
	m.advertiseAddr = MergeHostPort(opt.Host, opt.Addr)
	if len(opt.Peers) == 0 {
		opt.Peers = []string{m.advertiseAddr}
	}

	// Create a slice of peers with their HTTP address set instead.
	httpPeers, err := SetPortSlice(opt.Peers, opt.Addr)
	if err != nil {
		return fmt.Errorf("set port slice: %s", err)
	}
	m.peers = httpPeers

	// Initialise the default client using the peer list
	os.Setenv("DISCOVERD", strings.Join(opt.Peers, ","))
	discoverd.DefaultClient = discoverd.NewClient()

	// if there is a discoverd process already running on this
	// address perform a deployment by starting a proxy DNS server
	// and shutting down the old discoverd job
	var deploy *dd.Deployment
	var targetLogIndex dt.TargetLogIndex

	target := fmt.Sprintf("http://%s:1111", opt.Host)
	m.logger.Println("checking for existing discoverd process at", target)
	if err := discoverd.NewClientWithURL(target).Ping(target); err == nil {
		m.logger.Println("discoverd responding at", target, "taking over")

		deploy, err = dd.NewDeployment("discoverd")
		if err != nil {
			return err
		}
		m.logger.Println("Created deployment")
		if err := deploy.MarkPerforming(m.advertiseAddr, 60); err != nil {
			return err
		}
		m.logger.Println("marked", m.advertiseAddr, "as performing in deployent")
		addr, resolvers := waitHostDNSConfig()
		if opt.DNSAddr != "" {
			addr = opt.DNSAddr
		}
		if len(opt.Recursors) > 0 {
			resolvers = opt.Recursors
		}
		m.logger.Println("starting proxy DNS server")
		if err := m.openDNSServer(addr, resolvers); err != nil {
			return fmt.Errorf("Failed to start DNS server: %s", err)
		}
		m.logger.Printf("discoverd listening for DNS on %s", addr)

		targetLogIndex, err = discoverd.NewClientWithURL(target).Shutdown(target)
		if err != nil {
			return err
		}
		// Sleep for 2x the election timeout.
		// This is to work around an issue with hashicorp/raft that can allow us to be elected with
		// no log entries, hence truncating the log and losing all data!
		time.Sleep(2 * time.Second)
	} else {
		m.logger.Println("failed to contact existing discoverd server, starting up without takeover")
		m.logger.Println("err:", err)
	}

	// Open listener.
	ln, err := net.Listen("tcp4", opt.Addr)
	if err != nil {
		return err
	}
	m.ln = keepalive.Listener(ln)

	// Open mux
	m.mux = mux.New(m.ln)
	go m.mux.Serve()

	m.dataDir = opt.DataDir

	// if the advertise addr is not in the peer list we are proxying
	proxying := true
	for _, addr := range m.peers {
		if addr == m.advertiseAddr {
			proxying = false
			break
		}
	}

	if proxying {
		// Notify user that we're proxying if the store wasn't initialized.
		m.logger.Println("advertised address not in peer set, joining as proxy")
	} else {
		// Open store if we are not proxying.
		if err := m.openStore(); err != nil {
			return fmt.Errorf("Failed to open store: %s", err)
		}
	}

	// Wait for the store to catchup before switching to local store if we are doing a deployment
	if m.store != nil && targetLogIndex.LastIndex > 0 {
		for m.store.LastIndex() < targetLogIndex.LastIndex {
			m.logger.Println("Waiting for store to catchup, current:", m.store.LastIndex(), "target:", targetLogIndex.LastIndex)
			time.Sleep(100 * time.Millisecond)
		}
	}

	// If we already started the DNS server as part of a deployment above,
	// and we have an initialized store, just switch from the proxy store
	// to the initialized store.
	//
	// Else if we have a DNS address, start a DNS server right away.
	//
	// Otherwise wait for the host network to come up and then start a DNS
	// server.
	if m.dnsServer != nil && m.store != nil {
		m.dnsServer.SetStore(m.store)
	} else if opt.DNSAddr != "" {
		if err := m.openDNSServer(opt.DNSAddr, opt.Recursors); err != nil {
			return fmt.Errorf("Failed to start DNS server: %s", err)
		}
		m.logger.Printf("discoverd listening for DNS on %s", opt.DNSAddr)
	} else if opt.WaitNetDNS {
		go func() {
			addr, resolvers := waitHostDNSConfig()
			m.mu.Lock()
			if err := m.openDNSServer(addr, resolvers); err != nil {
				log.Fatalf("Failed to start DNS server: %s", err)
			}
			m.mu.Unlock()
			m.logger.Printf("discoverd listening for DNS on %s", addr)

			// Notify webhook.
			if opt.Notify != "" {
				m.Notify(opt.Notify, addr)
			}
		}()
	}

	if err := m.openHTTPServer(); err != nil {
		return fmt.Errorf("Failed to start HTTP server: %s", err)
	}

	if deploy != nil {
		if err := deploy.MarkDone(m.advertiseAddr); err != nil {
			return err
		}
		m.logger.Println("marked", m.advertiseAddr, "as done in deployment")
	}

	// Notify user that the servers are listening.
	m.logger.Printf("discoverd listening for HTTP on %s", opt.Addr)

	// Wait for leadership.
	if err := m.waitForLeader(IndefiniteTimeout); err != nil {
		return err
	}

	// Notify URL that discoverd is running.
	httpAddr := ln.Addr().String()
	host, port, _ := net.SplitHostPort(httpAddr)
	if host == "0.0.0.0" {
		httpAddr = net.JoinHostPort(os.Getenv("EXTERNAL_IP"), port)
	}
	m.Notify(opt.Notify, opt.DNSAddr)
	go func() {
		for {
			hb, err := discoverd.AddServiceAndRegister("discoverd", httpAddr)
			if err != nil {
				m.logger.Println("failed to register service/instance, retrying in 5 seconds:", err)
				time.Sleep(5 * time.Second)
				continue
			}
			m.mu.Lock()
			m.hb = hb
			m.mu.Unlock()
			break
		}
	}()
	return nil
}
Beispiel #3
0
// Run executes the program.
func (m *Main) Run(args ...string) error {
	// Create logger.
	m.logger = log.New(m.Stdout, "", log.LstdFlags)

	// Parse command line flags.
	opt, err := m.ParseFlags(args...)
	if err != nil {
		return err
	}

	// Set up advertised address and default peer set.
	advertiseAddr := MergeHostPort(opt.Host, opt.Addr)
	if len(opt.Peers) == 0 {
		opt.Peers = []string{advertiseAddr}
	}

	// Create a slice of peers with their HTTP address set instead.
	httpPeers, err := SetPortSlice(opt.Peers, opt.Addr)
	if err != nil {
		return fmt.Errorf("set port slice: %s", err)
	}

	// if there is a discoverd process already running on this
	// address perform a deployment by starting a proxy DNS server
	// and shutting down the old discoverd job
	var deploy *dd.Deployment
	var shutdownInfo dt.ShutdownInfo

	target := fmt.Sprintf("http://%s:1111", opt.Host)
	m.logger.Println("checking for existing discoverd process at", target)
	if err := discoverd.NewClientWithHTTP(target, &http.Client{}).Ping(); err == nil {
		m.logger.Println("discoverd responding at", target, "taking over")

		// update DISCOVERD environment variable so that the default
		// discoverd client connects to the instance we intend to replace.
		os.Setenv("DISCOVERD", target)
		discoverd.DefaultClient = discoverd.NewClient()

		deploy, err = dd.NewDeployment("discoverd")
		if err != nil {
			return err
		}
		m.logger.Println("Created deployment")
		if err := deploy.MarkPerforming(advertiseAddr, 60); err != nil {
			return err
		}
		m.logger.Println("marked", advertiseAddr, "as performing in deployent")
		addr, resolvers := waitHostDNSConfig()
		if opt.DNSAddr != "" {
			addr = opt.DNSAddr
		}
		if len(opt.Recursors) > 0 {
			resolvers = opt.Recursors
		}
		m.logger.Println("starting proxy DNS server")
		if err := m.openDNSServer(addr, resolvers, httpPeers); err != nil {
			return fmt.Errorf("Failed to start DNS server: %s", err)
		}
		m.logger.Printf("discoverd listening for DNS on %s", addr)

		shutdownInfo, err = discoverd.NewClientWithURL(target).Shutdown()
		if err != nil {
			return err
		}
	} else {
		m.logger.Println("failed to contact existing discoverd server, starting up without takeover")
	}

	// Open listener.
	ln, err := net.Listen("tcp4", opt.Addr)
	if err != nil {
		return err
	}
	m.ln = ln

	// Multiplex listener to store and http api.
	storeLn, httpLn := server.Mux(ln)

	// Open store if we are not proxying.
	if err := m.openStore(opt.DataDir, storeLn, advertiseAddr, opt.Peers); err != nil {
		return fmt.Errorf("Failed to open store: %s", err)
	}

	// Notify user that we're proxying if the store wasn't initialized.
	if m.store == nil {
		fmt.Fprintln(m.Stderr, "advertised address not in peer set, joining as proxy")
	}

	// Wait for the store to catchup before switching to local store if we are doing a deployment
	if m.store != nil && shutdownInfo.LastIndex > 0 {
		for m.store.LastIndex() < shutdownInfo.LastIndex {
			m.logger.Println("Waiting for store to catchup, current:", m.store.LastIndex(), "target:", shutdownInfo.LastIndex)
			time.Sleep(100 * time.Millisecond)
		}
	}

	// If we already started the DNS server as part of a deployment above,
	// and we have an initialized store, just switch from the proxy store
	// to the initialized store.
	//
	// Else if we have a DNS address, start a DNS server right away.
	//
	// Otherwise wait for the host network to come up and then start a DNS
	// server.
	if m.dnsServer != nil && m.store != nil {
		m.dnsServer.SetStore(m.store)
	} else if opt.DNSAddr != "" {
		if err := m.openDNSServer(opt.DNSAddr, opt.Recursors, httpPeers); err != nil {
			return fmt.Errorf("Failed to start DNS server: %s", err)
		}
		m.logger.Printf("discoverd listening for DNS on %s", opt.DNSAddr)
	} else if opt.WaitNetDNS {
		go func() {
			addr, resolvers := waitHostDNSConfig()
			if err := m.openDNSServer(addr, resolvers, httpPeers); err != nil {
				log.Fatalf("Failed to start DNS server: %s", err)
			}
			m.logger.Printf("discoverd listening for DNS on %s", addr)

			// Notify webhook.
			if opt.Notify != "" {
				m.Notify(opt.Notify, "", addr)
			}
		}()
	}

	if err := m.openHTTPServer(httpLn, opt.Peers); err != nil {
		return fmt.Errorf("Failed to start HTTP server: %s", err)
	}

	if deploy != nil {
		if err := deploy.MarkDone(advertiseAddr); err != nil {
			return err
		}
		m.logger.Println("marked", advertiseAddr, "as done in deployment")
	}

	// Notify user that the servers are listening.
	m.logger.Printf("discoverd listening for HTTP on %s", opt.Addr)

	// FIXME(benbjohnson): Join to cluster.

	// Wait for leadership.
	if err := m.waitForLeader(LeaderTimeout); err != nil {
		return err
	}

	// Notify URL that discoverd is running.
	httpAddr := ln.Addr().String()
	host, port, _ := net.SplitHostPort(httpAddr)
	if host == "0.0.0.0" {
		httpAddr = net.JoinHostPort(os.Getenv("EXTERNAL_IP"), port)
	}
	m.Notify(opt.Notify, "http://"+httpAddr, opt.DNSAddr)
	go discoverd.NewClientWithURL("http://"+httpAddr).AddServiceAndRegister("discoverd", httpAddr)

	return nil
}