func (c *timerTestProjectorClient) sendSync(instances []*protobuf.Instance) { logging.Infof("timerTestProjectorClient.sendSync() ") if len(instances) != 1 { util.TT.Fatal("timerTestProjectorClient.sendSync(): More than one index instance sent to fake projector") } for _, inst := range instances { if inst.GetIndexInstance().GetDefinition().GetDefnID() == uint64(406) { p := util.NewFakeProjector(manager.COORD_MAINT_STREAM_PORT) go p.Run(c.donech) payloads := make([]*common.VbKeyVersions, 0, 2000) // send StreamBegin for all vbuckets for i := 0; i < manager.NUM_VB; i++ { payload := common.NewVbKeyVersions("Default", uint16(i) /* vb */, 1, 10) kv := common.NewKeyVersions(1, []byte("document-name"), 1) kv.AddStreamBegin() kv.AddSync() payload.AddKeyVersions(kv) payloads = append(payloads, payload) } payload := common.NewVbKeyVersions("Default", 10, 1, 10) kv := common.NewKeyVersions(100, []byte("document-name"), 1) kv.AddSync() payload.AddKeyVersions(kv) payloads = append(payloads, payload) // send payload logging.Infof("****** runTimerTestReceiver() sending the first sync message") if err := p.Client.SendKeyVersions(payloads, true); err != nil { util.TT.Fatal(err) } payloads = make([]*common.VbKeyVersions, 0, 200) payload = common.NewVbKeyVersions("Default", 10, 1, 10) kv = common.NewKeyVersions(406, []byte("document-name"), 1) kv.AddSync() payload.AddKeyVersions(kv) payloads = append(payloads, payload) // send payload logging.Infof("****** runTimerTestReceiver() sending the second sync message") if err := p.Client.SendKeyVersions(payloads, true); err != nil { util.TT.Fatal(err) } } } }
func (c *streamEndTestProjectorClient) sendSync(timestamps []*protobuf.TsVbuuid) { logging.Infof("streamEndTestProjectorClient.sendSync() ") if len(timestamps) != 1 { util.TT.Fatal("streamEndTestProjectorClient.sendSync(): More than one timestamp sent to fake projector. Num = %v", len(timestamps)) } seqno, _, _, _, err := timestamps[0].Get(uint16(10)) if err != nil { util.TT.Fatal(err) } p := util.NewFakeProjector(manager.COORD_MAINT_STREAM_PORT) go p.Run(c.donech) payloads := make([]*common.VbKeyVersions, 0, 2000) payload := common.NewVbKeyVersions("Default", 10 /* vb */, 1, 10) kv := common.NewKeyVersions(seqno, []byte("document-name"), 1) kv.AddStreamBegin() kv.AddSync() payload.AddKeyVersions(kv) payloads = append(payloads, payload) // send payload if err := p.Client.SendKeyVersions(payloads, true); err != nil { util.TT.Fatal(err) } }
func TestAddDeletion(t *testing.T) { kv := kvDeletions() vbno, vbuuid, nMuts := uint16(10), uint64(1000), 10 vb := common.NewVbKeyVersions("default", vbno, vbuuid, nMuts) addKeyVersions(vb, []*common.KeyVersions{kv}, 1, nMuts) testKeyVersions(t, vb) }
func BenchmarkAddUpsertDeletionDecode(b *testing.B) { benchmarkMutationDecode(b, func() *common.VbKeyVersions { kv := kvUpsertDeletions() vbno, vbuuid, nMuts := uint16(10), uint64(1000), 10 vb := common.NewVbKeyVersions("default", vbno, vbuuid, nMuts) addKeyVersions(vb, []*common.KeyVersions{kv}, 1, nMuts) return vb }) }
func TestNewStreamEnd(t *testing.T) { seqno, docid, maxCount := uint64(10), []byte("document-name"), 10 kv := common.NewKeyVersions(seqno, docid, maxCount) kv.AddStreamEnd() // add it to VbKeyVersions vbno, vbuuid, nMuts := uint16(10), uint64(1000), 10 vb := common.NewVbKeyVersions("default", vbno, vbuuid, nMuts) vb.AddKeyVersions(kv) testKeyVersions(t, vb) }
func TestAddDropData(t *testing.T) { seqno, docid, maxCount := uint64(10), []byte(nil), 1 kv := common.NewKeyVersions(seqno, docid, maxCount) kv.AddDropData() // add it to VbKeyVersions vbno, vbuuid, nMuts := uint16(10), uint64(1000), 10 vb := common.NewVbKeyVersions("default", vbno, vbuuid, nMuts) vb.AddKeyVersions(kv) testKeyVersions(t, vb) }
func BenchmarkAddSyncDecode(b *testing.B) { benchmarkMutationDecode(b, func() *common.VbKeyVersions { seqno, docid, maxCount := uint64(10), []byte(nil), 1 kv := common.NewKeyVersions(seqno, docid, maxCount) kv.AddSync() vbno, vbuuid, nMuts := uint16(10), uint64(1000), 10 vb := common.NewVbKeyVersions("default", vbno, vbuuid, nMuts) vb.AddKeyVersions(kv) return vb }) }
func BenchmarkAddStreamEndDecode(b *testing.B) { benchmarkMutationDecode(b, func() *common.VbKeyVersions { seqno, docid, maxCount := uint64(10), []byte("document-name"), 10 kv := common.NewKeyVersions(seqno, docid, maxCount) kv.AddStreamEnd() // add it to VbKeyVersions vbno, vbuuid, nMuts := uint16(10), uint64(1000), 10 vb := common.NewVbKeyVersions("default", vbno, vbuuid, nMuts) vb.AddKeyVersions(kv) return vb }) }
// addKeyVersions, add a mutation's keyversions to buffer. func (b *endpointBuffers) addKeyVersions( bucket string, vbno uint16, vbuuid uint64, kv *c.KeyVersions) { if kv != nil && kv.Length() > 0 { uuid := c.StreamID(bucket, vbno) if _, ok := b.vbs[uuid]; !ok { nMuts := 16 // to avoid reallocs. b.vbs[uuid] = c.NewVbKeyVersions(bucket, vbno, vbuuid, nMuts) } b.vbs[uuid].AddKeyVersions(kv) } }
func TestStreamBegin(t *testing.T) { maxBuckets, maxvbuckets, mutChanSize := 2, 8, 1000 logging.SetLogLevel(logging.Silent) // start server appch := make(chan interface{}, mutChanSize) prefix := "indexer.dataport." config := c.SystemConfig.SectionConfig(prefix, true /*trim*/) daemon, err := NewServer(addr, maxvbuckets, config, appch) if err != nil { t.Fatal(err) } // start client flags := transport.TransportFlag(0).SetProtobuf() prefix = "projector.dataport.client." config = c.SystemConfig.SectionConfig(prefix, true /*trim*/) client, _ := NewClient( "cluster", "backfill", addr, flags, maxvbuckets, config) vbmaps := makeVbmaps(maxvbuckets, maxBuckets) // vbmaps for i := 0; i < maxBuckets; i++ { if err := client.SendVbmap(vbmaps[i]); err != nil { t.Fatal(err) } } // test a live StreamBegin bucket, vbno, vbuuid := "default0", uint16(maxvbuckets), uint64(1111) uuid := c.StreamID(bucket, vbno) vals, err := client.Getcontext() if err != nil { t.Fatal(err) } vbChans := vals[0].(map[string]chan interface{}) if _, ok := vbChans[uuid]; ok { t.Fatal("duplicate id") } vb := c.NewVbKeyVersions(bucket, vbno, vbuuid, 1) seqno, docid, maxCount := uint64(10), []byte("document-name"), 10 kv := c.NewKeyVersions(seqno, docid, maxCount) kv.AddStreamBegin() vb.AddKeyVersions(kv) err = client.SendKeyVersions([]*c.VbKeyVersions{vb}, true) client.Getcontext() // syncup if err != nil { t.Fatal(err) } else if _, ok := vbChans[uuid]; !ok { fmt.Printf("%v %v\n", len(vbChans), uuid) t.Fatal("failed StreamBegin") } client.Close() daemon.Close() }
func constructVbKeyVersions(bucket string, seqno, nVbs, nMuts, nIndexes int) []*c.VbKeyVersions { vbs := make([]*c.VbKeyVersions, 0, nVbs) for i := 0; i < nVbs; i++ { // for N vbuckets vbno, vbuuid := uint16(i), uint64(i*10) vb := c.NewVbKeyVersions(bucket, vbno, vbuuid, nMuts) for j := 0; j < nMuts; j++ { kv := c.NewKeyVersions(uint64(seqno+j), []byte("Bourne"), nIndexes) for k := 0; k < nIndexes; k++ { key := fmt.Sprintf("bangalore%v", k) oldkey := fmt.Sprintf("varanasi%v", k) kv.AddUpsert(uint64(k), []byte(key), []byte(oldkey)) } vb.AddKeyVersions(kv) } vbs = append(vbs, vb) } return vbs }
func (c *deleteTestProjectorClient) sendSync() { logging.Infof("deleteTestProjectorClient.sendSync() server %v", c.server) p := util.NewFakeProjector(manager.COORD_MAINT_STREAM_PORT) go p.Run(c.donech) // create an array of KeyVersions payloads := make([]*common.VbKeyVersions, 0, 4000) delete_test_once.Do(func() { logging.Infof("deleteTestProjectorClient.sendSync() sending streamBegin %v", c.server) // send StreamBegin for all vbuckets for i := 0; i < manager.NUM_VB; i++ { if i != 10 && i != 11 { payload := common.NewVbKeyVersions("Default", uint16(i) /* vb */, 1, 10) kv := common.NewKeyVersions(1, []byte("document-name"), 1) kv.AddStreamBegin() kv.AddSync() payload.AddKeyVersions(kv) payloads = append(payloads, payload) } } for i := 0; i < manager.NUM_VB; i++ { if i != 12 && i != 13 { payload := common.NewVbKeyVersions("Defaultxx", uint16(i) /* vb */, 1, 10) kv := common.NewKeyVersions(1, []byte("document-name"), 1) kv.AddStreamBegin() kv.AddSync() payload.AddKeyVersions(kv) payloads = append(payloads, payload) } } }) // bucket <Default>, node <127.0.0.1> -> vb 10, seqno 401 // bucket <Default>, node <127.0.0.2> -> vb 11, seqno 402 // bucket <Defaultxx>, node <127.0.0.1> -> vb 12, seqno 403 // bucket <Defaultxx>, node <127.0.0.2> -> vb 13, seqno 404 for _, inst := range delete_test_status { seqno := 0 vb := 0 bucket := inst.GetIndexInstance().GetDefinition().GetBucket() if bucket == "Default" { if c.server == "127.0.0.1" { seqno = 401 vb = 10 } else if c.server == "127.0.0.2" { seqno = 402 vb = 11 } } else if bucket == "Defaultxx" { if c.server == "127.0.0.1" { seqno = 403 vb = 12 } else if c.server == "127.0.0.2" { seqno = 404 vb = 13 } } logging.Infof("deleteTestProjectorClient.sendSync() for node %v and bucket %v vbucket %v seqno %d", c.server, bucket, vb, seqno) // Create Sync Message payload := common.NewVbKeyVersions(bucket, uint16(vb), 1, 10) kv := common.NewKeyVersions(uint64(seqno), []byte("document-name"), 1) kv.AddStreamBegin() kv.AddSync() payload.AddKeyVersions(kv) payloads = append(payloads, payload) } // Send payload if len(payloads) != 0 { logging.Infof("deleteTestProjectorClient.sendSync() sending payloads to stream manager for %s", c.server) err := p.Client.SendKeyVersions(payloads, true) if err != nil { util.TT.Fatal(err) } } }