func (c *Collector) Rcv(m beehive.Msg, ctx beehive.RcvContext) error {
res := m.Data().(StatResult)
glog.V(2).Infof("Stat results: %+v", res)
matrix := ctx.Dict(matrixDict)
key := res.Switch.Key()
v, err := matrix.Get(key)
if err != nil {
return fmt.Errorf("No such switch in matrix: %+v", res)
}
c.poller.query <- StatQuery{res.Switch}
sw := v.(SwitchStats)
stat, ok := sw[res.Flow]
sw[res.Flow] = res.Bytes
glog.V(2).Infof("Previous stats: %+v, Now: %+v", stat, res.Bytes)
if !ok || res.Bytes-stat > c.delta {
glog.Infof("Found an elephent flow: %+v, %+v, %+v", res, stat,
ctx.Hive().ID())
ctx.Emit(MatrixUpdate(res))
}
matrix.Put(key, sw)
return nil
}
func (r *registry) ApplyConfChange(cc raftpb.ConfChange, gn raft.GroupNode) (
err error) {
r.m.Lock()
defer r.m.Unlock()
glog.V(2).Infof("%v applies conf change %#v for %v", r, cc, gn.Node)
switch cc.Type {
case raftpb.ConfChangeAddNode:
if gn.Node != cc.NodeID {
glog.Fatalf("invalid data in the config change: %v != %v", gn.Node,
cc.NodeID)
}
if gn.Data != nil {
hi := HiveInfo{
ID: gn.Node,
Addr: gn.Data.(string),
}
r.addHive(hi)
glog.V(2).Infof("%v adds hive %v@%v", r, hi.ID, hi.Addr)
}
case raftpb.ConfChangeRemoveNode:
r.delHive(cc.NodeID)
glog.V(2).Infof("%v deletes hive %v", r, cc.NodeID)
}
return nil
}
func (h *hive) Start() error {
h.status = hiveStarted
h.registerSignals()
h.startRaftNode()
if err := h.listen(); err != nil {
glog.Errorf("%v cannot start listener: %v", h, err)
h.Stop()
return err
}
if err := h.raftBarrier(); err != nil {
glog.Fatalf("error when joining the cluster: %v", err)
}
glog.V(2).Infof("%v is in sync with the cluster", h)
h.startQees()
h.reloadState()
glog.V(2).Infof("%v starts message loop", h)
dataCh := h.dataCh.out()
for h.status == hiveStarted {
select {
case m := <-dataCh:
h.handleMsg(m.msg)
case cmd := <-h.ctrlCh:
h.handleCmd(cmd)
}
}
return nil
}
func (of *of10Driver) handlePacketIn(in of10.PacketIn, c *ofConn) error {
inPort := in.InPort()
// Ignore packet-ins on switch specific ports.
if inPort > uint16(of10.PP_MAX) {
glog.V(2).Infof("ignoring packet-in on %v", inPort)
return nil
}
port, ok := of.ofPorts[inPort]
if !ok {
return fmt.Errorf("of10Driver: port not found %v", inPort)
}
glog.V(2).Infof("packet received: %v", in)
nomIn := nom.PacketIn{
Node: c.node.UID(),
InPort: port.UID(),
BufferID: nom.PacketBufferID(in.BufferId()),
}
nomIn.Packet = nom.Packet(in.Data())
c.ctx.Emit(nomIn)
return nil
}
func (b *bee) ProcessStatusChange(sch interface{}) {
switch ev := sch.(type) {
case raft.LeaderChanged:
glog.V(2).Infof("%v recevies leader changed event %#v", b, ev)
if ev.New == Nil {
// TODO(soheil): when we switch to nil during a campaign, shouldn't we
// just change the colony?
return
}
oldc := b.colony()
oldi, err := b.hive.bee(oldc.Leader)
if err != nil {
glog.Fatalf("%v cannot find leader: %v", b, err)
}
if oldi.Hive == ev.New {
glog.V(2).Infof("%v has no need to change %v", b, oldc)
return
}
newc := oldc.DeepCopy()
if oldc.Leader != Nil {
newc.Leader = Nil
newc.AddFollower(oldc.Leader)
}
newi := b.fellowBeeOnHive(ev.New)
newc.DelFollower(newi.ID)
newc.Leader = newi.ID
b.setColony(newc)
go b.processCmd(cmdRefreshRole{})
if ev.New != b.hive.ID() {
return
}
b.setTerm(ev.Term)
go func() {
// FIXME(): add raft term to make sure it's versioned.
glog.V(2).Infof("%v is the new leader of %v", b, oldc)
up := updateColony{
Term: ev.Term,
Old: oldc,
New: newc,
}
// TODO(soheil): should we have a max retry?
_, err := b.hive.node.ProposeRetry(hiveGroup, up,
b.hive.config.RaftElectTimeout(), -1)
if err != nil {
glog.Errorf("%v cannot update its colony: %v", b, err)
}
}()
// TODO(soheil): add health checks here and recruit if needed.
}
}
func (b *bee) CommitTx() error {
// No need to replicate and/or persist the transaction.
if !b.app.persistent() || b.detached {
glog.V(2).Infof("%v commits in memory transaction", b)
b.commitTxBothLayers()
return nil
}
glog.V(2).Infof("%v commits persistent transaction", b)
return b.replicate()
}
func doHandleEchoRequest(req of.Header, res of.Header, c *ofConn) error {
glog.V(2).Infof("Received echo request from %v", c.node)
res.SetXid(req.Xid())
err := c.WriteHeaders([]of.Header{res})
if err != nil {
return err
}
c.Flush()
glog.V(2).Infof("Sent echo reply to %v", c.node)
return nil
}
func (of *of10Driver) handlePacketIn(in of10.PacketIn, c *ofConn) error {
inPort := in.InPort()
// Ignore packet-ins on switch specific ports.
if inPort > uint16(of10.PP_MAX) {
glog.V(2).Infof("ignoring packet-in on %v", inPort)
return nil
}
port, ok := of.ofPorts[inPort]
if !ok {
return fmt.Errorf("of10Driver: port not found %v", inPort)
}
if glog.V(2) {
glog.Infof("packet received: %v", in)
}
nomIn := nom.PacketIn{
Node: c.node.UID(),
InPort: port.UID(),
BufferID: nom.PacketBufferID(in.BufferId()),
}
nomIn.Packet = nom.Packet(in.Data())
c.ctx.Emit(nomIn)
//c.ctx.Emit(in)
//buf := make([]byte, 32)
//out := of10.NewPacketOutWithBuf(buf)
//out.Init()
//out.SetBufferId(in.BufferId())
//out.SetInPort(in.InPort())
//bcast := of10.NewActionOutput()
//bcast.SetPort(uint16(of10.PP_FLOOD))
//out.AddActions(bcast.ActionHeader)
//if in.BufferId() == 0xFFFFFFFF {
//for _, d := range in.Data() {
//out.AddData(d)
//}
//} else {
//out.SetBufferId(in.BufferId())
//}
//c.wCh <- out.Header
//if err := c.WriteHeader(out.Header); err != nil {
//return fmt.Errorf("Error in writing a packet out: %v", err)
//}
return nil
}
func (b *bee) replicate() error {
glog.V(2).Infof("%v replicates transaction", b)
b.Lock()
if b.stateL2 != nil {
err := b.commitTxL2()
b.stateL2 = nil
if err != nil && err != state.ErrNoTx {
b.Unlock()
return err
}
}
if b.stateL1.TxStatus() != state.TxOpen {
b.Unlock()
return state.ErrNoTx
}
stx := b.stateL1.Tx()
if len(stx.Ops) == 0 {
err := b.commitTxL1()
b.Unlock()
return err
}
b.Unlock()
if err := b.maybeRecruitFollowers(); err != nil {
return err
}
msgs := make([]*msg, len(b.msgBufL1))
copy(msgs, b.msgBufL1)
tx := tx{
Tx: stx,
Msgs: msgs,
}
ctx, cnl := context.WithTimeout(context.Background(),
10*b.hive.config.RaftElectTimeout())
defer cnl()
commit := commitTx{
Tx: tx,
Term: b.term(),
}
if _, err := b.hive.node.Propose(ctx, b.group(), commit); err != nil {
glog.Errorf("%v cannot replicate the transaction: %v", b, err)
return err
}
glog.V(2).Infof("%v successfully replicates transaction", b)
return nil
}
func (of *of12Driver) handlePacketIn(in of12.PacketIn, c *ofConn) error {
m := in.Match()
if m.Type() == uint16(of12.PMT_STANDARD) {
glog.Warningf("standard matches are not supported")
return nil
}
var inPort uint32
hasInPort := false
xm, _ := of12.ToOXMatch(in.Match())
for _, f := range xm.Fields() {
if of12.IsOxmInPort(f) {
xp, _ := of12.ToOxmInPort(f)
inPort = xp.InPort()
hasInPort = true
}
}
if !hasInPort {
glog.V(2).Infof("packet in does not have an input port")
return nil
}
// Ignore packet-ins on switch specific ports.
if inPort > uint32(of12.PP_MAX) {
glog.V(2).Infof("ignoring packet-in on %v", inPort)
return nil
}
port, ok := of.ofPorts[inPort]
if !ok {
return fmt.Errorf("of12Driver: port not found %v", inPort)
}
if glog.V(2) {
glog.Infof("packet received: %v", in)
}
nomIn := nom.PacketIn{
Node: c.node.UID(),
InPort: port.UID(),
BufferID: nom.PacketBufferID(in.BufferId()),
}
nomIn.Packet = nom.Packet(in.Data())
c.ctx.Emit(nomIn)
return nil
}
func (h *arpPktInHandler) Rcv(msg bh.Msg, ctx bh.RcvContext) error {
pin := msg.Data().(nom.PacketIn)
p := gopacket.NewPacket([]byte(pin.Packet), layers.LayerTypeEthernet, gopacket.Default)
etherlayer := p.Layer(layers.LayerTypeEthernet)
if etherlayer == nil {
return nil
}
e, _ := etherlayer.(*layers.Ethernet)
if e.EthernetType != layers.EthernetTypeARP {
return nil
}
host, _, err := decodeARP([]byte(pin.Packet))
host.Node = pin.Node
if err != nil {
glog.Errorf("ARP decoding error: %v", err)
return err
}
glog.V(2).Infof("Host detected: %v", host)
ctx.Emit(nom.HostConnected(host))
return nil
}
func (q *qee) handleUnicastMsg(mh msgAndHandler) {
glog.V(2).Infof("unicast msg: %v", mh.msg)
b, ok := q.beeByID(mh.msg.To())
if !ok {
info, err := q.hive.registry.bee(mh.msg.To())
if err != nil {
glog.Errorf("cannot find bee %v", mh.msg.To())
}
if q.isLocalBee(info) {
glog.Fatalf("%v cannot find local bee %v", q, mh.msg.To())
}
if b, ok = q.beeByID(info.ID); !ok {
if b, err = q.newProxyBee(info); err != nil {
glog.Errorf("%v cannnot find remote bee %v", q, mh.msg.To())
return
}
}
}
if mh.handler == nil && !b.detached && !b.proxy {
glog.Fatalf("handler is nil for message %v", mh.msg)
}
b.enqueMsg(mh)
}
func (b *bee) handoff(to uint64) error {
if !b.app.persistent() {
return b.handoffNonPersistent(to)
}
c := b.colony()
if !c.IsFollower(to) {
return fmt.Errorf("%v is not a follower of %v", to, b)
}
if _, err := b.qee.sendCmdToBee(to, cmdSync{}); err != nil {
return err
}
ch := make(chan error)
go func() {
// TODO(soheil): use context with deadline here.
_, err := b.qee.sendCmdToBee(to, cmdCampaign{})
ch <- err
}()
t := b.hive.config.RaftElectTimeout()
time.Sleep(t)
if _, err := b.hive.node.ProposeRetry(c.ID, noOp{}, t, 10); err != nil {
glog.Errorf("%v cannot sync raft: %v", b, err)
}
if b.isFollower(b.ID()) {
glog.V(2).Infof("%v successfully handed off leadership to %v", b, to)
b.becomeFollower()
}
return <-ch
}
func (r *registry) doApply(req interface{}) (interface{}, error) {
glog.V(2).Infof("%v applies: %#v", r, req)
switch req := req.(type) {
case noOp:
return nil, nil
case newHiveID:
return r.newHiveID(), nil
case allocateBeeIDs:
return r.allocBeeIDs(req)
case addBee:
return nil, r.addBee(BeeInfo(req))
case delBee:
return nil, r.delBee(uint64(req))
case moveBee:
return nil, r.moveBee(req)
case updateColony:
return nil, r.updateColony(req)
case lockMappedCell:
return r.lockCell(req)
case transferCells:
return nil, r.transfer(req)
case batchReq:
return r.handleBatch(req), nil
}
glog.Errorf("%v cannot handle %v", r, req)
return nil, ErrUnsupportedRequest
}
func (d *Driver) Rcv(m beehive.Msg, ctx beehive.RcvContext) error {
if m.NoReply() {
return nil
}
q, ok := m.Data().(StatQuery)
if !ok {
return nil
}
s, ok := d.switches[q.Switch]
if !ok {
return fmt.Errorf("No switch stored in the driver: %+v", s)
}
for i, f := range s.Flows {
f.Bytes += uint64(rand.Intn(maxSpike))
s.Flows[i] = f
glog.V(2).Infof("Emitting stat result for %+v", f)
ctx.Emit(StatResult{q, f.Flow, f.Bytes})
}
d.switches[q.Switch] = s
return nil
}
func (g *group) applyConfChange(e raftpb.Entry) error {
var cc raftpb.ConfChange
pbutil.MustUnmarshal(&cc, e.Data)
glog.V(2).Infof("%v applies conf change %v: %#v", g, e.Index, cc)
if len(cc.Context) == 0 {
g.stateMachine.ApplyConfChange(cc, GroupNode{})
return nil
}
if id, req, err := g.node.decReq(cc.Context); err == nil {
if gn, ok := req.Data.(GroupNode); ok {
res := Response{ID: id}
res.Err = g.stateMachine.ApplyConfChange(cc, gn)
g.node.line.call(res)
return nil
}
}
var gn GroupNode
if err := bhgob.Decode(&gn, cc.Context); err != nil {
glog.Fatalf("%v cannot decode config change: %v", g, err)
}
if gn.Node != cc.NodeID {
glog.Fatalf("invalid config change: %v != %v", gn.Node, cc.NodeID)
}
g.stateMachine.ApplyConfChange(cc, gn)
return nil
}
func (h *hive) stopQees() {
glog.Infof("%v is stopping qees...", h)
qs := make(map[*qee]bool)
for _, mhs := range h.qees {
for _, mh := range mhs {
qs[mh.q] = true
}
}
stopCh := make(chan cmdResult)
for q := range qs {
q.ctrlCh <- newCmdAndChannel(cmdStop{}, h.ID(), q.app.Name(), 0, stopCh)
glog.V(3).Infof("waiting on a qee: %v", q)
stopped := false
tries := 5
for !stopped {
select {
case res := <-stopCh:
_, err := res.get()
if err != nil {
glog.Errorf("error in stopping a qee: %v", err)
}
stopped = true
case <-time.After(1 * time.Second):
if tries--; tries < 0 {
glog.Infof("giving up on qee %v", q)
stopped = true
continue
}
glog.Infof("still waiting for a qee %v...", q)
}
}
}
}
func (b *bee) createGroup() error {
c := b.colony()
if c.IsNil() || c.ID == Nil {
return fmt.Errorf("%v is in no colony", b)
}
cfg := raft.GroupConfig{
ID: c.ID,
Name: b.String(),
StateMachine: b,
Peers: b.peers(),
DataDir: b.statePath(),
SnapCount: 1024,
FsyncTick: b.hive.config.RaftFsyncTick,
ElectionTicks: b.hive.config.RaftElectTicks,
HeartbeatTicks: b.hive.config.RaftHBTicks,
MaxInFlights: b.hive.config.RaftInFlights,
MaxMsgSize: b.hive.config.RaftMaxMsgSize,
}
if err := b.hive.node.CreateGroup(context.TODO(), cfg); err != nil {
return err
}
if err := b.raftBarrier(); err != nil {
return err
}
b.enableEmit()
glog.V(2).Infof("%v started its raft node", b)
return nil
}
func (h *newLinkHandler) Rcv(msg bh.Msg, ctx bh.RcvContext) error {
l := nom.Link(msg.Data().(NewLink))
n, _ := nom.ParsePortUID(l.From)
d := ctx.Dict(nodeDict)
k := string(n)
v, err := d.Get(k)
if err != nil {
return nil
}
np := v.(nodePortsAndLinks)
if oldl, ok := np.linkFrom(l.From); ok {
if oldl.UID() == l.UID() {
return nil
}
np.removeLink(oldl)
ctx.Emit(nom.LinkDeleted(oldl))
}
glog.V(2).Infof("Link detected %v", l)
ctx.Emit(nom.LinkAdded(l))
np.L = append(np.L, l)
return d.Put(k, np)
}
func peersInfo(addrs []string) map[uint64]HiveInfo {
if len(addrs) == 0 {
return nil
}
ch := make(chan []HiveInfo, len(addrs))
for _, a := range addrs {
go func(a string) {
s, err := getHiveState(a)
if err != nil {
glog.Errorf("cannot communicate with %v: %v", a, err)
return
}
ch <- s.Peers
}(a)
}
// Return the first one.
hives := <-ch
glog.V(2).Infof("found live hives: %v", hives)
infos := make(map[uint64]HiveInfo)
for _, h := range hives {
infos[h.ID] = h
}
return infos
}
func (b *bee) Apply(req interface{}) (interface{}, error) {
b.Lock()
defer b.Unlock()
switch r := req.(type) {
case commitTx:
if b.txTerm < r.Term {
b.txTerm = r.Term
} else if r.Term < b.txTerm {
return nil, ErrOldTx
}
glog.V(2).Infof("%v commits %v", b, r)
leader := b.isLeader()
if b.stateL2 != nil {
b.stateL2 = nil
glog.Errorf("%v has an L2 transaction", b)
}
if b.stateL1.TxStatus() == state.TxOpen {
if !leader {
glog.Errorf("%v is a follower and has an open transaction", b)
}
b.resetTx(b.stateL1, &b.msgBufL1)
}
if err := b.stateL1.Apply(r.Tx.Ops); err != nil {
return nil, err
}
if leader && b.emitInRaft {
for _, msg := range r.Tx.Msgs {
msg.MsgFrom = b.beeID
glog.V(2).Infof("%v emits %#v", b, msg)
}
b.throttle(r.Tx.Msgs)
}
return nil, nil
case noOp:
return nil, nil
}
glog.Errorf("%v cannot handle %v", b, req)
return nil, ErrUnsupportedRequest
}
func (h nodeConnectedHandler) Rcv(msg bh.Msg, ctx bh.RcvContext) error {
nc := msg.Data().(nom.NodeConnected)
ddict := ctx.Dict(driversDict)
k := string(nc.Node.ID)
n := nodeDrivers{
Node: nc.Node,
}
if v, err := ddict.Get(k); err == nil {
n = v.(nodeDrivers)
}
if _, ok := n.driver(nc.Driver); ok {
return fmt.Errorf("driver %v reconnects to %v", nc.Driver, n.Node)
}
gdict := ctx.Dict(genDict)
gen := uint64(0)
if v, err := gdict.Get("gen"); err == nil {
gen = v.(uint64)
}
gen++
db := nc.Driver.BeeID
if len(n.Drivers) == 0 {
nc.Driver.Role = nom.DriverRoleMaster
ctx.Emit(nom.NodeJoined(nc.Node))
glog.V(2).Infof("%v connected to master controller", nc.Node)
} else {
nc.Driver.Role = nom.DriverRoleSlave
glog.V(2).Infof("%v connected to slave controller", nc.Node)
}
n.Drivers = append(n.Drivers, driverInfo{
Driver: nc.Driver,
LastSeen: time.Now(),
})
ctx.SendToBee(nom.ChangeDriverRole{
Node: nc.Node.UID(),
Role: nc.Driver.Role,
Generation: gen,
}, db)
gdict.Put("gen", gen)
return ddict.Put(k, n)
}
func (g *group) fsync() error {
glog.V(2).Infof("%v syncing disk storage", g)
if err := g.diskStorage.Sync(); err != nil {
glog.Errorf("%v cannot sync disk storage: %v", g, err)
return err
}
return nil
}
func (r *registry) delBee(id uint64) error {
glog.V(2).Infof("%v removes bee %v", r, id)
if _, ok := r.Bees[id]; !ok {
return ErrNoSuchBee
}
delete(r.Bees, id)
return nil
}
func (c *ofConn) handshake() (ofDriver, error) {
hdr, err := c.ReadHeader()
if err != nil {
return nil, err
}
h, err := of.ToHello(hdr)
if err != nil {
return nil, err
}
glog.V(2).Info("%v received hello from a switch with OFv%v", c.ctx,
h.Version())
version := of.OPENFLOW_1_0
if h.Version() >= uint8(of.OPENFLOW_1_2) {
version = of.OPENFLOW_1_2
}
h.SetVersion(uint8(version))
if err = c.WriteHeader(h.Header); err != nil {
return nil, err
}
c.Flush()
glog.V(2).Info("%v sent hello to the switch", c.ctx)
var driver ofDriver
switch version {
case of.OPENFLOW_1_0:
driver = &of10Driver{}
case of.OPENFLOW_1_2:
driver = &of12Driver{}
}
if err = driver.handshake(c); err != nil {
return nil, err
}
if c.node.ID == nom.NodeID(0) {
return nil, errors.New("ofConn: invalid node after handshake")
}
return driver, nil
}
func (c *rpcClient) sendCmd(cm cmd) (res interface{}, err error) {
glog.V(3).Infof("%v sends %v", c, cm)
r := make([]cmdResult, 1)
err = c.cmd.Call("rpcServer.ProcessCmd", []cmd{cm}, &r)
if err != nil {
return
}
return r[0].Data, r[0].Err
}
func (r *registry) ProcessStatusChange(sch interface{}) {
switch ev := sch.(type) {
case raft.LeaderChanged:
if ev.New != r.HiveID {
return
}
glog.V(2).Infof("hive %v is the new leader", r.HiveID)
}
}
func (b *bee) bufferOrEmit(m *msg) {
dicts, msgs := b.currentState()
if dicts.TxStatus() != state.TxOpen {
b.throttle([]*msg{m})
return
}
glog.V(2).Infof("buffers msg %+v in tx", m)
*msgs = append(*msgs, m)
}
func (c *rpcClient) sendRaft(batch *raft.Batch, r raft.Reporter) (err error) {
glog.V(3).Infof("%v sends a raft batch", c)
var dummy bool
if batch.Priority == raft.High {
err = c.prio.Call("rpcServer.ProcessRaft", batch, &dummy)
} else {
err = c.raft.Call("rpcServer.ProcessRaft", batch, &dummy)
}
report(err, batch, r)
return err
}
func (r *registry) addHive(info HiveInfo) error {
glog.V(2).Infof("%v sets hive %v's address to %v", r, info.ID, info.Addr)
for _, h := range r.Hives {
if h.Addr == info.Addr && h.ID != info.ID {
return fmt.Errorf("%v has duplicate address %v for hives %v and %v", r,
info.Addr, info.ID, h.ID)
}
}
r.Hives[info.ID] = info
return nil
}