コード例 #1
0
ファイル: consumer.go プロジェクト: mdmarek/grid
func (a *ConsumerActor) Running() dfa.Letter {
	ticker := time.NewTicker(30 * time.Second)
	defer ticker.Stop()

	w := condition.NewNameWatch(a.grid.Etcd(), a.grid.Name(), a.flow.Name(), "finished")
	defer w.Stop()

	n := 0
	finished := w.WatchUntil(ring.New(a.flow.NewContextualName("producer"), a.conf.NrProducers))
	for {
		select {
		case <-a.exit:
			return Exit
		case <-a.chaos.C:
			return Failure
		case <-ticker.C:
			if err := a.started.Alive(); err != nil {
				return Failure
			}
		case <-finished:
			if err := a.SendCounts(); err != nil {
				return SendFailure
			} else {
				return EverybodyFinished
			}
		case err := <-w.WatchError():
			log.Printf("%v: error: %v", a, err)
			return Failure
		case m := <-a.rx.Msgs():
			switch m := m.(type) {
			case DataMsg:
				a.state.Counts[m.Producer]++
				n++
				if n%1000000 == 0 {
					log.Printf("%v: received: %v", a.ID(), n)
				}
				if n%1000 == 0 {
					if err := a.SendCounts(); err != nil {
						return SendFailure
					}
				}
			}
		}
	}
}
コード例 #2
0
ファイル: producer.go プロジェクト: mdmarek/grid
func (a *ProducerActor) Running() dfa.Letter {
	ticker := time.NewTicker(30 * time.Second)
	defer ticker.Stop()

	a.state = NewProducerState()
	s := condition.NewState(a.grid.Etcd(), 30*time.Minute, a.grid.Name(), a.flow.Name(), "state", a.ID())
	defer s.Stop()

	if err := s.Init(a.state); err != nil {
		if _, err := s.Fetch(a.state); err != nil {
			return FetchStateFailure
		}
	}
	log.Printf("%v: running with state: %v, index: %v", a.ID(), a.state, s.Index())

	// Make some random length string data.
	data := NewDataMaker(a.conf.MsgSize, a.conf.MsgCount-a.state.SentMessages)
	defer data.Stop()

	r := ring.New(a.flow.NewContextualName("consumer"), a.conf.NrConsumers)
	start := time.Now()
	for {
		select {
		case <-a.exit:
			if _, err := s.Store(a.state); err != nil {
				log.Printf("%v: failed to save state: %v", a, err)
			}
			return Exit
		case <-a.chaos.C:
			if _, err := s.Store(a.state); err != nil {
				log.Printf("%v: failed to save state: %v", a, err)
			}
			return Failure
		case <-ticker.C:
			if err := a.started.Alive(); err != nil {
				return Failure
			}
			if _, err := s.Store(a.state); err != nil {
				return Failure
			}
		case <-data.Done():
			if err := a.tx.Flush(); err != nil {
				return SendFailure
			}
			if err := a.tx.Send(a.flow.NewContextualName("leader"), &ResultMsg{Producer: a.ID(), Count: a.state.SentMessages, From: a.ID(), Duration: a.state.Duration}); err != nil {
				return SendFailure
			}
			if _, err := s.Store(a.state); err != nil {
				log.Printf("%v: failed to save state: %v", a, err)
			}
			return IndividualFinished
		case d := <-data.Next():
			if a.state.SentMessages%100 == 0 {
				a.state.Duration += time.Now().Sub(start).Seconds()
				start = time.Now()
			}
			if a.state.SentMessages%10000 == 0 {
				if _, err := s.Store(a.state); err != nil {
					log.Printf("%v: failed to save state: %v", a, err)
					return Failure
				}
			}
			if err := a.tx.SendBuffered(r.ByInt(a.state.SentMessages), &DataMsg{Producer: a.ID(), Data: d}); err != nil {
				if _, err := s.Store(a.state); err != nil {
					log.Printf("%v: failed to save state: %v", a, err)
				}
				return SendFailure
			}
			a.state.SentMessages++
		}
	}
}
コード例 #3
0
ファイル: main.go プロジェクト: mdmarek/grid
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	flag.Parse()

	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()

	etcdservers := strings.Split(*etcdconnect, ",")
	natsservers := strings.Split(*natsconnect, ",")

	conf := &Conf{
		GridName:    "flowgrid",
		MsgSize:     *msgsize,
		MsgCount:    *msgcount,
		NrProducers: *producers,
		NrConsumers: *consumers,
	}

	hostname, err := os.Hostname()
	if err != nil {
		log.Fatalf("error: failed to discover hostname: %v", err)
	}

	m, err := newActorMaker(conf)
	if err != nil {
		log.Fatalf("error: failed to make actor maker: %v", err)
	}

	g := grid.New(conf.GridName, etcdservers, natsservers, m)

	exit, err := g.Start()
	if err != nil {
		log.Fatalf("error: failed to start grid: %v", err)
	}

	j := condition.NewJoin(g.Etcd(), 30*time.Second, g.Name(), "hosts", hostname)
	err = j.Join()
	if err != nil {
		log.Fatalf("error: failed to regester: %v", err)
	}
	defer j.Exit()
	go func() {
		ticker := time.NewTicker(15 * time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-exit:
				return
			case <-ticker.C:
				err := j.Alive()
				if err != nil {
					log.Fatalf("error: failed to report liveness: %v", err)
				}
			}
		}
	}()

	w := condition.NewCountWatch(g.Etcd(), g.Name(), "hosts")
	defer w.Stop()

	started := w.WatchUntil(*nodes)
	select {
	case <-exit:
		log.Printf("Shutting down, grid exited")
		return
	case <-w.WatchError():
		log.Fatalf("error: failed to watch other hosts join: %v", err)
	case <-started:
	}

	for i := 0; i < *flows; i++ {
		flow := NewFlow(i)

		rp := ring.New(flow.NewContextualName("producer"), conf.NrProducers)
		for _, def := range rp.ActorDefs() {
			def.DefineType("producer")
			def.Define("flow", flow.Name())
			err := g.StartActor(def)
			if err != nil {
				log.Fatalf("error: failed to start: %v, due to: %v", def, err)
			}
		}

		rc := ring.New(flow.NewContextualName("consumer"), conf.NrConsumers)
		for _, def := range rc.ActorDefs() {
			def.DefineType("consumer")
			def.Define("flow", flow.Name())
			err := g.StartActor(def)
			if err != nil {
				log.Fatalf("error: failed to start: %v, due to: %v", def, err)
			}
		}

		def := grid.NewActorDef(flow.NewContextualName("leader"))
		def.DefineType("leader")
		def.Define("flow", flow.Name())
		err = g.StartActor(def)
		if err != nil {
			log.Fatalf("error: failed to start: %v, due to: %v", "leader", err)
		}

		time.Sleep(5 * time.Second)
	}

	go func() {
		sig := make(chan os.Signal, 1)
		signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
		select {
		case <-sig:
			log.Printf("shutting down")
			g.Stop()
		case <-exit:
		}
	}()

	<-exit
	log.Println("shutdown complete")
}