func (a *LeaderActor) Running() dfa.Letter {
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
a.state = NewLeaderState()
s := condition.NewState(a.grid.Etcd(), 10*time.Minute, a.grid.Name(), a.flow.Name(), "state", a.ID())
defer s.Stop()
if err := s.Init(a.state); err != nil {
if _, err := s.Fetch(a.state); err != nil {
return FetchStateFailure
}
}
log.Printf("%v: running with state: %v, index: %v", a.ID(), a.state, s.Index())
w := condition.NewCountWatch(a.grid.Etcd(), a.grid.Name(), a.flow.Name(), "finished")
defer w.Stop()
finished := w.WatchUntil(a.conf.NrConsumers + a.conf.NrProducers)
for {
select {
case <-a.exit:
if _, err := s.Store(a.state); err != nil {
log.Printf("%v: failed to save state: %v", a, err)
}
return Exit
case <-a.chaos.C:
if _, err := s.Store(a.state); err != nil {
log.Printf("%v: failed to save state: %v", a, err)
}
return Failure
case <-ticker.C:
if err := a.started.Alive(); err != nil {
return Failure
}
if _, err := s.Store(a.state); err != nil {
return Failure
}
case <-finished:
return EverybodyFinished
case err := <-w.WatchError():
log.Printf("%v: error: %v", a, err)
return Failure
case m := <-a.rx.Msgs():
switch m := m.(type) {
case ResultMsg:
if strings.Contains(m.From, "producer") {
a.state.ProducerCounts[m.Producer] = m.Count
a.state.ProducerDurations[m.Producer] = m.Duration
}
if strings.Contains(m.From, "consumer") {
a.state.ConsumerCounts[m.Producer] += m.Count
}
}
}
}
}
func (a *LeaderActor) Starting() dfa.Letter {
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
time.Sleep(3 * time.Second)
j := condition.NewJoin(a.grid.Etcd(), 2*time.Minute, a.grid.Name(), a.flow.Name(), "started", a.ID())
if err := j.Rejoin(); err != nil {
return Failure
}
a.started = j
w := condition.NewCountWatch(a.grid.Etcd(), a.grid.Name(), a.flow.Name(), "started")
defer w.Stop()
f := condition.NewNameWatch(a.grid.Etcd(), a.grid.Name(), a.flow.Name(), "finished")
defer f.Stop()
started := w.WatchUntil(a.conf.NrConsumers + a.conf.NrProducers + 1)
finished := f.WatchUntil(a.flow.NewContextualName("leader"))
for {
select {
case <-a.exit:
return Exit
case <-a.chaos.C:
return Failure
case <-ticker.C:
if err := a.started.Alive(); err != nil {
return Failure
}
case <-started:
return EverybodyStarted
case <-finished:
return EverybodyFinished
}
}
}
func (a *LeaderActor) Finishing() dfa.Letter {
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
j := condition.NewJoin(a.grid.Etcd(), 10*time.Minute, a.grid.Name(), a.flow.Name(), "finished", a.ID())
if err := j.Rejoin(); err != nil {
return Failure
}
a.finished = j
w := condition.NewCountWatch(a.grid.Etcd(), a.grid.Name(), a.flow.Name(), "finished")
defer w.Stop()
finished := w.WatchUntil(a.conf.NrConsumers + a.conf.NrProducers + 1)
for {
select {
case <-a.exit:
return Exit
case <-a.chaos.C:
return Failure
case <-ticker.C:
if err := a.started.Alive(); err != nil {
return Failure
}
if err := a.finished.Alive(); err != nil {
return Failure
}
case <-finished:
a.started.Exit()
a.finished.Alive()
return EverybodyFinished
case err := <-w.WatchError():
log.Printf("%v: error: %v", a, err)
return Failure
}
}
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
flag.Parse()
go func() {
log.Println(http.ListenAndServe("localhost:6060", nil))
}()
etcdservers := strings.Split(*etcdconnect, ",")
natsservers := strings.Split(*natsconnect, ",")
conf := &Conf{
GridName: "flowgrid",
MsgSize: *msgsize,
MsgCount: *msgcount,
NrProducers: *producers,
NrConsumers: *consumers,
}
hostname, err := os.Hostname()
if err != nil {
log.Fatalf("error: failed to discover hostname: %v", err)
}
m, err := newActorMaker(conf)
if err != nil {
log.Fatalf("error: failed to make actor maker: %v", err)
}
g := grid.New(conf.GridName, etcdservers, natsservers, m)
exit, err := g.Start()
if err != nil {
log.Fatalf("error: failed to start grid: %v", err)
}
j := condition.NewJoin(g.Etcd(), 30*time.Second, g.Name(), "hosts", hostname)
err = j.Join()
if err != nil {
log.Fatalf("error: failed to regester: %v", err)
}
defer j.Exit()
go func() {
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
for {
select {
case <-exit:
return
case <-ticker.C:
err := j.Alive()
if err != nil {
log.Fatalf("error: failed to report liveness: %v", err)
}
}
}
}()
w := condition.NewCountWatch(g.Etcd(), g.Name(), "hosts")
defer w.Stop()
started := w.WatchUntil(*nodes)
select {
case <-exit:
log.Printf("Shutting down, grid exited")
return
case <-w.WatchError():
log.Fatalf("error: failed to watch other hosts join: %v", err)
case <-started:
}
for i := 0; i < *flows; i++ {
flow := NewFlow(i)
rp := ring.New(flow.NewContextualName("producer"), conf.NrProducers)
for _, def := range rp.ActorDefs() {
def.DefineType("producer")
def.Define("flow", flow.Name())
err := g.StartActor(def)
if err != nil {
log.Fatalf("error: failed to start: %v, due to: %v", def, err)
}
}
rc := ring.New(flow.NewContextualName("consumer"), conf.NrConsumers)
for _, def := range rc.ActorDefs() {
def.DefineType("consumer")
def.Define("flow", flow.Name())
err := g.StartActor(def)
if err != nil {
log.Fatalf("error: failed to start: %v, due to: %v", def, err)
}
}
def := grid.NewActorDef(flow.NewContextualName("leader"))
def.DefineType("leader")
def.Define("flow", flow.Name())
err = g.StartActor(def)
if err != nil {
log.Fatalf("error: failed to start: %v, due to: %v", "leader", err)
}
time.Sleep(5 * time.Second)
}
go func() {
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
select {
case <-sig:
log.Printf("shutting down")
g.Stop()
case <-exit:
}
}()
<-exit
log.Println("shutdown complete")
}