コード例 #1
0
ファイル: init.go プロジェクト: ably-forks/flynn
func monitor(port host.Port, container *ContainerInit, env map[string]string, log log15.Logger) (discoverd.Heartbeater, error) {
	config := port.Service
	client := discoverd.NewClientWithURL(env["DISCOVERD"])
	client.Logger = logger.New("component", "discoverd")

	if config.Create {
		// TODO: maybe reuse maybeAddService() from the client
		log.Info("creating service")
		if err := client.AddService(config.Name, nil); err != nil {
			if !hh.IsObjectExistsError(err) {
				log.Error("error creating service", "err", err)
				return nil, fmt.Errorf("something went wrong with discoverd: %s", err)
			}
		}
	}
	inst := &discoverd.Instance{
		Addr:  fmt.Sprintf("%s:%v", env["EXTERNAL_IP"], port.Port),
		Proto: port.Proto,
	}
	// add discoverd.EnvInstanceMeta if present
	for k, v := range env {
		if _, ok := discoverd.EnvInstanceMeta[k]; !ok {
			continue
		}
		if inst.Meta == nil {
			inst.Meta = make(map[string]string)
		}
		inst.Meta[k] = v
	}

	// no checker, but we still want to register a service
	if config.Check == nil {
		log.Info("registering instance", "instance", inst)
		return client.RegisterInstance(config.Name, inst)
	}

	var check health.Check
	switch config.Check.Type {
	case "tcp":
		check = &health.TCPCheck{Addr: inst.Addr}
	case "http", "https":
		check = &health.HTTPCheck{
			URL:        fmt.Sprintf("%s://%s%s", config.Check.Type, inst.Addr, config.Check.Path),
			Host:       config.Check.Host,
			StatusCode: config.Check.Status,
			MatchBytes: []byte(config.Check.Match),
		}
	default:
		// unsupported checker type
		return nil, fmt.Errorf("unsupported check type: %s", config.Check.Type)
	}
	log.Info("adding healthcheck", "type", config.Check.Type, "interval", config.Check.Interval, "threshold", config.Check.Threshold)
	reg := health.Registration{
		Registrar: client,
		Service:   config.Name,
		Instance:  inst,
		Monitor: health.Monitor{
			Interval:  config.Check.Interval,
			Threshold: config.Check.Threshold,
			Logger:    log.New("component", "monitor"),
		}.Run,
		Check:  check,
		Logger: log,
	}

	if config.Check.KillDown {
		reg.Events = make(chan health.MonitorEvent)
		go func() {
			if config.Check.StartTimeout == 0 {
				config.Check.StartTimeout = 10 * time.Second
			}

			start := false
			lastStatus := health.MonitorStatusDown
			var mtx sync.Mutex

			maybeKill := func() {
				if lastStatus == health.MonitorStatusDown {
					log.Warn("killing the job")
					container.Signal(int(syscall.SIGKILL), &struct{}{})
				}
			}
			go func() {
				// ignore events for the first StartTimeout interval
				<-time.After(config.Check.StartTimeout)
				mtx.Lock()
				defer mtx.Unlock()
				maybeKill() // check if the app is down
				start = true
			}()

			for e := range reg.Events {
				log.Info("got health monitor event", "status", e.Status)
				mtx.Lock()
				lastStatus = e.Status
				if !start {
					mtx.Unlock()
					continue
				}
				maybeKill()
				mtx.Unlock()
			}
		}()
	}
	return reg.Register(), nil
}
コード例 #2
0
ファイル: watch_test.go プロジェクト: rikur/flynn
func (s *ClientSuite) TestWatchReconnect(c *C) {
	c.Skip("fix discoverd watch reconnect") // FIXME(benbjohnson)

	raftPort, err := testutil.RandomPort()
	c.Assert(err, IsNil)

	httpPort, err := testutil.RandomPort()
	c.Assert(err, IsNil)

	// clientA is used to register services and instances, and remains connected
	clientA, cleanup := testutil.SetupDiscoverd(c)
	defer cleanup()

	// clientB is connected to the server which will be restarted, and is used to
	// test that the watch generates the correct events after reconnecting
	clientB, killDiscoverd := testutil.BootDiscoverd(c, raftPort, httpPort)
	defer func() { killDiscoverd() }()

	// create a service with manual leader and some metadata
	service := "foo"
	config := &discoverd.ServiceConfig{LeaderType: discoverd.LeaderTypeManual}
	c.Assert(clientA.AddService(service, config), IsNil)
	serviceMeta := &discoverd.ServiceMeta{Data: []byte(`{"foo": "bar"}`)}
	c.Assert(clientA.Service(service).SetMeta(serviceMeta), IsNil)

	register := func(client *discoverd.Client, addr string, meta map[string]string) (discoverd.Heartbeater, *discoverd.Instance) {
		inst := &discoverd.Instance{Addr: addr, Proto: "tcp", Meta: meta}
		hb, err := client.RegisterInstance(service, inst)
		c.Assert(err, IsNil)
		return hb, inst
	}
	waitForEvent := func(events chan *discoverd.Event, addr string, kind discoverd.EventKind) {
		for {
			select {
			case e := <-events:
				if e.Kind == kind && (addr == "" || addr == e.Instance.Addr) {
					return
				}
			case <-time.After(10 * time.Second):
				c.Fatalf("timed out wating for %s event", kind)
			}
		}
	}
	waitForWatchState := func(ch chan discoverd.WatchState, state discoverd.WatchState) {
		for {
			select {
			case s := <-ch:
				if s == state {
					return
				}
			case <-time.After(10 * time.Second):
				c.Fatalf("timed out waiting for watch %s state", state)
			}
		}
	}

	// register three services
	register(clientA, ":1111", nil)
	hb2, _ := register(clientA, ":2222", map[string]string{"foo": "bar"})
	hb3, _ := register(clientA, ":3333", nil)

	// create watches using both clients so we can synchronize assertions
	eventsA := make(chan *discoverd.Event)
	watchA, err := clientA.Service(service).Watch(eventsA)
	c.Assert(err, IsNil)
	defer watchA.Close()
	waitForEvent(eventsA, "", discoverd.EventKindCurrent)

	eventsB := make(chan *discoverd.Event)
	watchB, err := clientB.Service(service).Watch(eventsB)
	c.Assert(err, IsNil)
	defer watchB.Close()
	waitForEvent(eventsB, "", discoverd.EventKindCurrent)

	// kill clientB's server and wait for the watch to disconnect
	stateCh := make(chan discoverd.WatchState)
	watchB.(*discoverd.Watch).SetStateChannel(stateCh)
	killDiscoverd()
	waitForWatchState(stateCh, discoverd.WatchStateDisconnected)

	// make some changes using clientA

	// change some metadata
	c.Assert(hb2.SetMeta(map[string]string{"foo": "baz"}), IsNil)
	waitForEvent(eventsA, ":2222", discoverd.EventKindUpdate)

	// register a new instance
	_, inst := register(clientA, ":4444", nil)
	waitForEvent(eventsA, ":4444", discoverd.EventKindUp)

	// set a new leader
	clientA.Service(service).SetLeader(inst.ID)
	waitForEvent(eventsA, ":4444", discoverd.EventKindLeader)

	// unregister an instance
	hb3.Close()
	waitForEvent(eventsA, ":3333", discoverd.EventKindDown)

	// update the service metadata
	serviceMeta.Data = []byte(`{"foo": "baz"}`)
	c.Assert(clientA.Service(service).SetMeta(serviceMeta), IsNil)
	waitForEvent(eventsA, "", discoverd.EventKindServiceMeta)

	// restart clientB's server and wait for the watch to reconnect
	_, killDiscoverd = testutil.RunDiscoverdServer(c, raftPort, httpPort)
	waitForWatchState(stateCh, discoverd.WatchStateConnected)

	type expectedEvent struct {
		Addr        string
		Kind        discoverd.EventKind
		ServiceMeta *discoverd.ServiceMeta
	}

	assertCurrent := func(events chan *discoverd.Event, expected []*expectedEvent) {
		count := 0
		isExpected := func(event *discoverd.Event) bool {
			for _, e := range expected {
				if e.Kind != event.Kind {
					continue
				}
				switch event.Kind {
				case discoverd.EventKindServiceMeta:
					if reflect.DeepEqual(event.ServiceMeta, e.ServiceMeta) {
						return true
					}
				default:
					if event.Instance != nil && event.Instance.Addr == e.Addr {
						return true
					}
				}
			}
			return false
		}
		for {
			select {
			case event := <-events:
				if event.Kind == discoverd.EventKindCurrent {
					if count != len(expected) {
						c.Fatalf("expected %d events, got %d", len(expected), count)
					}
					return
				}
				if !isExpected(event) {
					c.Fatalf("unexpected event: %+v", event)
				}
				count++
			case <-time.After(10 * time.Second):
				c.Fatal("timed out waiting for events")
			}
		}
	}

	// check watchB emits missed events
	assertCurrent(eventsB, []*expectedEvent{
		{Addr: ":2222", Kind: discoverd.EventKindUpdate},
		{Addr: ":4444", Kind: discoverd.EventKindUp},
		{Addr: ":4444", Kind: discoverd.EventKindLeader},
		{Kind: discoverd.EventKindServiceMeta, ServiceMeta: serviceMeta},
		{Addr: ":3333", Kind: discoverd.EventKindDown},
	})
}