func (p *Peer) handleReconRqstPoly(rp *ReconRqstPoly) *msgProgress {
remoteSize := rp.Size
points := p.ptree.Points()
remoteSamples := rp.Samples
node, err := p.ptree.Node(rp.Prefix)
if err == ErrNodeNotFound {
return &msgProgress{err: ErrReconRqstPolyNotFound}
}
localSamples := node.SValues()
localSize := node.Size()
remoteSet, localSet, err := p.solve(
remoteSamples, localSamples, remoteSize, localSize, points)
if errgo.Cause(err) == cf.ErrLowMBar {
p.log(GOSSIP).Info("ReconRqstPoly: low MBar")
if node.IsLeaf() || node.Size() < (p.settings.ThreshMult*p.settings.MBar) {
p.logFields(GOSSIP, log.Fields{
"node": node.Key(),
}).Info("sending full elements")
elements, err := node.Elements()
if err != nil {
return &msgProgress{err: errgo.Mask(err)}
}
return &msgProgress{elements: cf.NewZSet(), messages: []ReconMsg{
&FullElements{ZSet: cf.NewZSet(elements...)}}}
} else {
err = errgo.Notef(err, "bs=%v leaf=%v size=%d", node.Key(), node.IsLeaf(), node.Size())
}
}
if err != nil {
p.logErr(GOSSIP, err).Info("ReconRqstPoly: sending SyncFail")
return &msgProgress{elements: cf.NewZSet(), messages: []ReconMsg{&SyncFail{}}}
}
p.logFields(GOSSIP, log.Fields{"localSet": localSet, "remoteSet": remoteSet}).Info("ReconRqstPoly: solved")
return &msgProgress{elements: remoteSet, messages: []ReconMsg{&Elements{ZSet: localSet}}}
}
func (s *ReconSuite) RunOneSided(c *gc.C, n int, serverHas bool, timeout time.Duration) {
ptree1, cleanup, err := s.Factory()
c.Assert(err, gc.IsNil)
defer cleanup()
ptree2, cleanup, err := s.Factory()
c.Assert(err, gc.IsNil)
defer cleanup()
expected := cf.NewZSet()
for i := 1; i < n; i++ {
z := cf.Zi(cf.P_SKS, 65537*i)
ptree2.Insert(z)
expected.Add(z)
}
port1, port2 := portPair(c)
var peer1Mode recon.PeerMode
var peer2Mode recon.PeerMode
if serverHas {
peer1Mode = recon.PeerModeGossipOnly
peer2Mode = recon.PeerModeServeOnly
} else {
peer1Mode = recon.PeerModeServeOnly
peer2Mode = recon.PeerModeGossipOnly
}
peer1 := s.newPeer(port1, port2, peer1Mode, ptree1)
peer2 := s.newPeer(port2, port1, peer2Mode, ptree2)
err = s.pollConvergence(c, peer1, peer2, expected, cf.NewZSet(), timeout)
c.Assert(err, gc.IsNil)
}
func (rwc *reconWithClient) sendRequest(p *Peer, req *requestEntry) error {
if req == nil {
return errgo.New("nil request")
}
var msg ReconMsg
if req.node.IsLeaf() || (req.node.Size() < p.settings.MBar) {
elements, err := req.node.Elements()
if err != nil {
return err
}
msg = &ReconRqstFull{
Prefix: req.key,
Elements: cf.NewZSet(elements...)}
} else {
msg = &ReconRqstPoly{
Prefix: req.key,
Size: req.node.Size(),
Samples: req.node.SValues()}
}
p.logFields(SERVE, log.Fields{"msg": msg}).Debug("sendRequest")
rwc.messages = append(rwc.messages, msg)
rwc.pushBottom(&bottomEntry{requestEntry: req})
return nil
}
func ReadZSet(r io.Reader) (*cf.ZSet, error) {
arr, err := ReadZZarray(r)
if err != nil {
return nil, err
}
zset := cf.NewZSet()
zset.AddSlice(arr)
return zset, nil
}
// Test sync with polynomial interpolation.
func (s *ReconSuite) TestPolySyncMBar(c *gc.C) {
ptree1, cleanup, err := s.Factory()
c.Assert(err, gc.IsNil)
defer cleanup()
ptree2, cleanup, err := s.Factory()
c.Assert(err, gc.IsNil)
defer cleanup()
onlyInPeer1 := cf.NewZSet()
// Load up peer 1 with items
for i := 1; i < 100; i++ {
ptree1.Insert(cf.Zi(cf.P_SKS, 65537*i))
}
// Four extra samples
for i := 1; i < 5; i++ {
z := cf.Zi(cf.P_SKS, 68111*i)
ptree1.Insert(z)
onlyInPeer1.Add(z)
}
root, _ := ptree1.Root()
c.Log("peer1:", recon.MustElements(root))
onlyInPeer2 := cf.NewZSet()
// Load up peer 2 with items
for i := 1; i < 100; i++ {
ptree2.Insert(cf.Zi(cf.P_SKS, 65537*i))
}
// One extra sample
for i := 1; i < 2; i++ {
z := cf.Zi(cf.P_SKS, 70001*i)
ptree2.Insert(z)
onlyInPeer2.Add(z)
}
root, _ = ptree2.Root()
c.Log("peer2:", recon.MustElements(root))
port1, port2 := portPair(c)
peer1 := s.newPeer(port1, port2, recon.PeerModeGossipOnly, ptree1)
peer2 := s.newPeer(port2, port1, recon.PeerModeServeOnly, ptree2)
err = s.pollConvergence(c, peer1, peer2, onlyInPeer2, onlyInPeer1, LongTimeout)
c.Assert(err, gc.IsNil)
}
func (p *Peer) handleReconRqstFull(rf *ReconRqstFull) *msgProgress {
var localset *cf.ZSet
node, err := p.ptree.Node(rf.Prefix)
if err == ErrNodeNotFound {
localset = cf.NewZSet()
} else if err != nil {
return &msgProgress{err: err}
} else {
elements, err := node.Elements()
if err != nil {
return &msgProgress{err: err}
}
localset = cf.NewZSet(elements...)
}
localNeeds := cf.ZSetDiff(rf.Elements, localset)
remoteNeeds := cf.ZSetDiff(localset, rf.Elements)
p.logFields(GOSSIP, log.Fields{
"localNeeds": localNeeds.Len(),
"remoteNeeds": remoteNeeds.Len(),
}).Info("ReconRqstFull")
return &msgProgress{elements: localNeeds, messages: []ReconMsg{&Elements{ZSet: remoteNeeds}}}
}
func (p *Peer) clientRecon(conn net.Conn, remoteConfig *Config) error {
w := bufio.NewWriter(conn)
respSet := cf.NewZSet()
defer func() {
p.sendItems(respSet.Items(), conn, remoteConfig)
}()
var pendingMessages []ReconMsg
for step := range p.interactWithServer(conn) {
if step.err != nil {
if step.err == ErrReconDone {
p.log(GOSSIP).Info("reconcilation done")
break
} else {
err := WriteMsg(w, &Error{&textMsg{Text: step.err.Error()}})
if err != nil {
p.logErr(GOSSIP, err).Error()
}
p.logErr(GOSSIP, step.err).Error("step error")
break
}
} else {
pendingMessages = append(pendingMessages, step.messages...)
if step.flush {
for _, msg := range pendingMessages {
err := WriteMsg(w, msg)
if err != nil {
return errgo.Mask(err)
}
}
pendingMessages = nil
err := w.Flush()
if err != nil {
return errgo.Mask(err)
}
}
}
p.log(GOSSIP).Debugf("add step: %v", step)
respSet.AddAll(step.elements)
p.log(GOSSIP).Infof("recover set now %d elements", respSet.Len())
}
return nil
}
func (rwc *reconWithClient) handleReply(p *Peer, msg ReconMsg, req *requestEntry) error {
rwc.Peer.logFields(SERVE, log.Fields{"msg": msg}).Debug("handleReply")
switch m := msg.(type) {
case *SyncFail:
if req.node.IsLeaf() {
return errgo.New("Syncfail received at leaf node")
}
rwc.Peer.log(SERVE).Debug("SyncFail: pushing children")
children, err := req.node.Children()
if err != nil {
return errgo.Mask(err)
}
for i, childNode := range children {
rwc.Peer.logFields(SERVE, log.Fields{"childNode": childNode.Key()}).Debug("push")
if i == 0 {
rwc.pushRequest(&requestEntry{key: childNode.Key(), node: childNode})
} else {
rwc.prependRequests(&requestEntry{key: childNode.Key(), node: childNode})
}
}
case *Elements:
rwc.rcvrSet.AddAll(m.ZSet)
case *FullElements:
elements, err := req.node.Elements()
if err != nil {
return err
}
local := cf.NewZSet(elements...)
localNeeds := cf.ZSetDiff(m.ZSet, local)
remoteNeeds := cf.ZSetDiff(local, m.ZSet)
elementsMsg := &Elements{ZSet: remoteNeeds}
rwc.Peer.logFields(SERVE, log.Fields{
"msg": elementsMsg,
}).Debug("handleReply: sending")
rwc.messages = append(rwc.messages, elementsMsg)
rwc.rcvrSet.AddAll(localNeeds)
default:
return errgo.Newf("unexpected message: %v", m)
}
return nil
}
func (s *PtreeSuite) TestJustOneKey(c *gc.C) {
root, err := s.ptree.Root()
c.Assert(err, gc.IsNil)
s.ptree.Insert(cf.Zs(cf.P_SKS, "224045810486609649306292620830306652473"))
expect := cf.NewZSet()
for _, sv := range []string{
"306467079064992673198834899522272784866",
"306467079064992673198834899522272784865",
"306467079064992673198834899522272784867",
"306467079064992673198834899522272784864",
"306467079064992673198834899522272784868",
"306467079064992673198834899522272784863"} {
expect.Add(cf.Zs(cf.P_SKS, sv))
}
c.Assert(err, gc.IsNil)
root, err = s.ptree.Root()
for _, sv := range root.SValues() {
c.Assert(expect.Has(sv), gc.Equals, true, gc.Commentf("Unexpected svalue: %v", sv))
expect.Remove(sv)
}
c.Assert(expect.Items(), gc.HasLen, 0)
}
func (p *Peer) interactWithServer(conn net.Conn) msgProgressChan {
out := make(msgProgressChan)
go func() {
defer close(out)
var resp *msgProgress
var n int
for (resp == nil || resp.err == nil) && n < maxRecoverSize {
p.setReadDeadline(conn, defaultTimeout)
msg, err := ReadMsg(conn)
if err != nil {
p.logErr(GOSSIP, err).Error("interact: read msg")
out <- &msgProgress{err: err}
return
}
p.logFields(GOSSIP, log.Fields{"msg": msg}).Debug("interact")
switch m := msg.(type) {
case *ReconRqstPoly:
resp = p.handleReconRqstPoly(m)
case *ReconRqstFull:
resp = p.handleReconRqstFull(m)
case *Elements:
p.logFields(GOSSIP, log.Fields{"nelements": m.ZSet.Len()}).Debug()
resp = &msgProgress{elements: m.ZSet}
case *Done:
resp = &msgProgress{err: ErrReconDone}
case *Flush:
resp = &msgProgress{elements: cf.NewZSet(), flush: true}
default:
resp = &msgProgress{err: errgo.Newf("unexpected message: %v", m)}
}
n += resp.elements.Len()
out <- resp
}
}()
return out
}
func (p *Peer) interactWithClient(conn net.Conn, remoteConfig *Config, bitstring *cf.Bitstring) error {
p.log(SERVE).Debug("interacting with client")
p.setReadDeadline(conn, defaultTimeout)
recon := reconWithClient{
Peer: p,
conn: conn,
bwr: bufio.NewWriter(conn),
rcvrSet: cf.NewZSet(),
}
root, err := p.ptree.Root()
if err != nil {
return err
}
defer func() {
p.sendItems(recon.rcvrSet.Items(), conn, remoteConfig)
}()
defer func() {
WriteMsg(recon.bwr, &Done{})
}()
recon.pushRequest(&requestEntry{node: root, key: bitstring})
for !recon.isDone() {
bottom := recon.topBottom()
p.logFields(SERVE, log.Fields{"bottom": bottom}).Debug("interact")
switch {
case bottom == nil:
req := recon.popRequest()
p.logFields(SERVE, log.Fields{
"popRequest": req,
}).Debug("interact: sending...")
err = recon.sendRequest(p, req)
if err != nil {
return err
}
case bottom.state == reconStateFlushEnded:
p.log(SERVE).Debug("interact: flush ended, popBottom")
recon.popBottom()
recon.flushing = false
case bottom.state == reconStateBottom:
p.logFields(SERVE, log.Fields{
"queueLength": len(recon.bottomQ),
}).Debug()
var msg ReconMsg
var hasMsg bool
// Set a small read timeout to simulate non-blocking I/O
p.setReadDeadline(conn, time.Millisecond)
if err != nil {
return errgo.Mask(err)
}
msg, nbErr := ReadMsg(conn)
hasMsg = (nbErr == nil)
// Restore blocking I/O
p.setReadDeadline(conn, defaultTimeout)
if err != nil {
return errgo.Mask(err)
}
if hasMsg {
recon.popBottom()
err = recon.handleReply(p, msg, bottom.requestEntry)
if err != nil {
return errgo.Mask(err)
}
} else if len(recon.bottomQ) > p.settings.MaxOutstandingReconRequests ||
len(recon.requestQ) == 0 {
if !recon.flushing {
err = recon.flushQueue()
if err != nil {
return errgo.Mask(err)
}
} else {
recon.popBottom()
p.setReadDeadline(conn, 3*time.Second)
msg, err = ReadMsg(conn)
if err != nil {
return errgo.Mask(err)
}
p.logFields(SERVE, log.Fields{"msg": msg}).Debug("reply")
err = recon.handleReply(p, msg, bottom.requestEntry)
if err != nil {
return errgo.Mask(err)
}
}
} else {
req := recon.popRequest()
err = recon.sendRequest(p, req)
if err != nil {
return err
}
}
default:
return errgo.New("failed to match expected patterns")
}
}
return nil
}
func (s *ReconSuite) pollRootConvergence(c *gc.C, peer1, peer2 *recon.Peer, ptree1, ptree2 recon.PrefixTree) error {
var t tomb.Tomb
t.Go(func() error {
defer peer1.Stop()
defer peer2.Stop()
var mu sync.Mutex
var zs1 *cf.ZSet = cf.NewZSet()
var zs2 *cf.ZSet = cf.NewZSet()
timer := time.NewTimer(LongTimeout)
peer1.SetMutatedFunc(func() {
mu.Lock()
root1, err := ptree1.Root()
c.Assert(err, gc.IsNil)
zs1 = cf.NewZSet(recon.MustElements(root1)...)
mu.Unlock()
})
peer2.SetMutatedFunc(func() {
mu.Lock()
root2, err := ptree2.Root()
c.Assert(err, gc.IsNil)
zs2 = cf.NewZSet(recon.MustElements(root2)...)
mu.Unlock()
})
POLLING:
for {
select {
case r1, ok := <-peer1.RecoverChan:
if !ok {
break POLLING
}
c.Logf("peer1 recover: %v", r1)
for _, zp := range r1.RemoteElements {
c.Assert(zp, gc.NotNil)
peer1.Insert(zp)
}
case r2, ok := <-peer2.RecoverChan:
if !ok {
break POLLING
}
c.Logf("peer2 recover: %v", r2)
for _, zp := range r2.RemoteElements {
c.Assert(zp, gc.NotNil)
peer2.Insert(zp)
}
case _ = <-timer.C:
return fmt.Errorf("timeout waiting for convergence")
default:
}
var done bool
mu.Lock()
done = zs1.Len() > 0 && zs1.Equal(zs2)
mu.Unlock()
if done {
c.Logf("peer1 has %q, peer2 has %q", zs1, zs2)
return nil
}
}
return fmt.Errorf("set reconciliation did not converge")
})
return t.Wait()
}
// Init configures the tree with default settings if not already set,
// and initializes the internal state with sample data points, root node, etc.
func (t *MemPrefixTree) Init() {
t.PTreeConfig = defaultPTreeConfig
t.points = cf.Zpoints(cf.P_SKS, t.NumSamples())
t.allElements = cf.NewZSet()
t.Create()
}
