Beispiel #1
0
func startBucket(cluster, bucketn string, rch chan []interface{}) int {
	defer func() {
		if r := recover(); r != nil {
			logging.Errorf("Recovered from panic %v", r)
			logging.Errorf(logging.StackTrace())
		}
	}()

	logging.Infof("Connecting with %q\n", bucketn)
	b, err := common.ConnectBucket(cluster, "default", bucketn)
	mf(err, "bucket")

	dcpConfig := map[string]interface{}{
		"genChanSize":  10000,
		"dataChanSize": 10000,
	}
	dcpFeed, err := b.StartDcpFeed("rawupr", uint32(0), 0xABCD, dcpConfig)
	mf(err, "- upr")

	vbnos := listOfVbnos(options.maxVbno)
	flogs, err := b.GetFailoverLogs(0xABCD, vbnos, dcpConfig)
	mf(err, "- dcp failoverlogs")
	if options.printflogs {
		printFlogs(vbnos, flogs)
	}
	go startDcp(dcpFeed, flogs)

	for {
		e, ok := <-dcpFeed.C
		if ok == false {
			logging.Infof("Closing for bucket %q\n", bucketn)
		}
		rch <- []interface{}{bucketn, e}
	}
}
Beispiel #2
0
func main() {
	logging.SetLogLevel(logging.Error)
	runtime.GOMAXPROCS(runtime.NumCPU())

	cmdOptions, args, fset, err := querycmd.ParseArgs(os.Args[1:])
	if err != nil {
		logging.Fatalf("%v", err)
		os.Exit(0)
	} else if cmdOptions.Help {
		usage(fset)
		os.Exit(0)
	} else if len(args) < 1 {
		logging.Fatalf("%v", "specify a command")
	}

	b, err := c.ConnectBucket(cmdOptions.Server, "default", "default")
	if err != nil {
		log.Fatal(err)
	}
	defer b.Close()
	maxvb, err := c.MaxVbuckets(b)
	if err != nil {
		log.Fatal(err)
	}

	config := c.SystemConfig.SectionConfig("queryport.client.", true)
	client, err := qclient.NewGsiClient(cmdOptions.Server, config)
	if err != nil {
		log.Fatal(err)
	}

	switch args[0] {
	case "sanity":
		err = doSanityTests(cmdOptions.Server, client)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error occured %v\n", err)
		}

	case "mb14786":
		err = doMB14786(cmdOptions.Server, client)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error occured %v\n", err)
		}

	case "mb13339":
		err = doMB13339(cmdOptions.Server, client)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error occured %v\n", err)
		}

	case "benchmark":
		doBenchmark(cmdOptions.Server, "localhost:8101")

	case "consistency":
		doConsistency(cmdOptions.Server, maxvb, client)
	}
	client.Close()
}
Beispiel #3
0
func numVbuckets(cluster, bucketn string) (numVb int) {
	b, err := common.ConnectBucket(cluster, "default" /*pooln*/, bucketn)
	if err != nil {
		log.Fatal(err)
	}
	defer b.Close()
	if numVb, err = common.MaxVbuckets(b); err != nil {
		log.Fatal(err)
	}
	return numVb
}
Beispiel #4
0
// - return couchbase SDK error if any.
func (p *Projector) doFailoverLog(
	request *protobuf.FailoverLogRequest, opaque uint16) ap.MessageMarshaller {

	response := &protobuf.FailoverLogResponse{}

	pooln := request.GetPool()
	bucketn := request.GetBucket()
	vbuckets := request.GetVbnos()

	// log this request.
	prefix := p.logPrefix
	fmsg := "%v ##%x doFailoverLog() {%q, %q, %v}\n"
	logging.Infof(fmsg, prefix, opaque, pooln, bucketn, vbuckets)
	defer logging.Infof("%v ##%x doFailoverLog() returns ...\n", prefix, opaque)

	bucket, err := c.ConnectBucket(p.clusterAddr, pooln, bucketn)
	if err != nil {
		logging.Errorf("%v ##%x ConnectBucket(): %v\n", prefix, opaque, err)
		response.Err = protobuf.NewError(err)
		return response
	}
	defer bucket.Close()

	protoFlogs := make([]*protobuf.FailoverLog, 0, len(vbuckets))
	vbnos := c.Vbno32to16(vbuckets)
	dcpConfig := map[string]interface{}{
		"genChanSize":  p.config["projector.dcp.genChanSize"].Int(),
		"dataChanSize": p.config["projector.dcp.dataChanSize"].Int(),
	}
	flogs, err := bucket.GetFailoverLogs(opaque, vbnos, dcpConfig)
	if err == nil {
		for vbno, flog := range flogs {
			vbuuids := make([]uint64, 0, len(flog))
			seqnos := make([]uint64, 0, len(flog))
			for _, x := range flog {
				vbuuids = append(vbuuids, x[0])
				seqnos = append(seqnos, x[1])
			}
			protoFlog := &protobuf.FailoverLog{
				Vbno:    proto.Uint32(uint32(vbno)),
				Vbuuids: vbuuids,
				Seqnos:  seqnos,
			}
			protoFlogs = append(protoFlogs, protoFlog)
		}
	} else {
		logging.Errorf("%v ##%x GetFailoverLogs(): %v\n", prefix, opaque, err)
		response.Err = protobuf.NewError(err)
		return response
	}
	response.Logs = protoFlogs
	return response
}
Beispiel #5
0
func bucketTs(cluster, bucketn string, numVb int) {
	b, err := common.ConnectBucket(cluster, "default" /*pooln*/, bucketn)
	if err != nil {
		log.Fatal(err)
	}
	defer b.Close()

	start := time.Now()
	for i := 0; i < options.trials; i++ {
		if _, _, err = common.BucketTs(b, numVb); err != nil {
			log.Fatal(err)
		}
	}
	durtn := time.Since(start) / time.Duration(options.trials)
	log.Printf("bucketTs: %v\n", durtn)
}
Beispiel #6
0
// - return couchbase SDK error if any.
func (p *Projector) doVbmapRequest(
	request *protobuf.VbmapRequest, opaque uint16) ap.MessageMarshaller {

	response := &protobuf.VbmapResponse{}

	pooln := request.GetPool()
	bucketn := request.GetBucket()
	kvaddrs := request.GetKvaddrs()

	// log this request.
	prefix := p.logPrefix
	fmsg := "%v ##%x doVbmapRequest() {%q, %q, %v}\n"
	logging.Infof(fmsg, prefix, pooln, bucketn, kvaddrs, opaque)
	defer logging.Infof("%v ##%x doVbmapRequest() returns ...\n", prefix, opaque)

	// get vbmap from bucket connection.
	bucket, err := c.ConnectBucket(p.clusterAddr, pooln, bucketn)
	if err != nil {
		logging.Errorf("%v ##%x ConnectBucket(): %v\n", prefix, opaque, err)
		response.Err = protobuf.NewError(err)
		return response
	}
	defer bucket.Close()

	bucket.Refresh()
	m, err := bucket.GetVBmap(kvaddrs)
	if err != nil {
		logging.Errorf("%v ##%x GetVBmap(): %v\n", prefix, opaque, err)
		response.Err = protobuf.NewError(err)
		return response
	}

	// compose response
	response.Kvaddrs = make([]string, 0, len(kvaddrs))
	response.Kvvbnos = make([]*protobuf.Vbuckets, 0, len(kvaddrs))
	for kvaddr, vbnos := range m {
		response.Kvaddrs = append(response.Kvaddrs, kvaddr)
		response.Kvvbnos = append(
			response.Kvvbnos, &protobuf.Vbuckets{Vbnos: c.Vbno16to32(vbnos)})
	}
	return response
}
Beispiel #7
0
// BucketTs will return the current vbucket-timestamp using STATS
// command.
func (c *GsiClient) BucketTs(bucketn string) (*TsConsistency, error) {
	b, err := common.ConnectBucket(c.cluster, "default" /*pooln*/, bucketn)
	if err != nil {
		return nil, err
	}
	defer b.Close()

	if c.maxvb == -1 {
		if c.maxvb, err = common.MaxVbuckets(b); err != nil {
			return nil, err
		}
	}
	seqnos, vbuuids, err := common.BucketTs(b, c.maxvb)
	if err != nil {
		return nil, err
	}
	vbnos := make([]uint16, c.maxvb)
	for i := range vbnos {
		vbnos[i] = uint16(i)
	}
	return NewTsConsistency(vbnos, seqnos, vbuuids), nil
}
func startBucket(cluster, bucketn string, ch chan *couchbase.UprFeed) int {
	defer func() {
		if r := recover(); r != nil {
			fmt.Printf("%s:\n%s\n", r, debug.Stack())
			common.StackTrace(string(debug.Stack()))
		}
	}()

	common.Infof("Connecting with %q\n", bucketn)
	b, err := common.ConnectBucket(cluster, "default", bucketn)
	mf(err, "bucket")

	uprFeed, err := b.StartUprFeed("rawupr", uint32(0))
	mf(err, "- upr")

	vbnos := listOfVbnos(options.maxVbno)

	flogs, err := b.GetFailoverLogs(vbnos)
	mf(err, "- upr failoverlogs")

	if options.printflogs {
		printFlogs(vbnos, flogs)
	}

	ch <- uprFeed

	go startUpr(uprFeed, flogs)

	for {
		e, ok := <-uprFeed.C
		if ok == false {
			common.Infof("Closing for bucket %q\n", bucketn)
		}
		rch <- []interface{}{bucketn, e}
	}
}
Beispiel #9
0
func doConsistency(
	cluster string, maxvb int, client *qclient.GsiClient) (err error) {

	b, err := common.ConnectBucket(cluster, "default", "beer-sample")
	if err != nil {
		log.Fatal(err)
	}
	defer b.Close()

	vbnos := make([]uint16, maxvb)
	for i := range vbnos {
		vbnos[i] = uint16(i)
	}

	// Drop index
	args := []string{"-type", "drop", "-bucket", "beer-sample", "-index", "index-city"}
	cmd, _, _, _ := querycmd.ParseArgs(args)
	querycmd.HandleCommand(client, cmd, true, os.Stdout)

	// Create index
	args = []string{
		"-type", "create", "-bucket", "beer-sample", "-index", "index-city",
		"-fields", "city",
	}
	cmd, _, _, err = querycmd.ParseArgs(args)
	if err != nil {
		log.Fatal(err)
	}
	querycmd.HandleCommand(client, cmd, true, os.Stdout)

	// Wait for index to come active.
	index, ok := querycmd.GetIndex(client, "beer-sample", "index-city")
	if !ok {
		log.Fatalf("cannot get definition ID")
	}
	defnID := uint64(index.Definition.DefnId)
	_, err = querycmd.WaitUntilIndexState(
		client, []uint64{defnID}, common.INDEX_STATE_ACTIVE,
		100 /*period*/, 20000 /*timeout*/)

	synch := make(chan bool, 1)
	// Get the latest seqnos,vbuuid and vbucket that contains `docid`.
	seqnos, vbuuids, vbno, vbuuid, seqno := setValueConst(b, maxvb, constDocValue1)
	equal := common.SecondaryKey(querycmd.Arg2Key(constEqualLookup1))
	equals := []common.SecondaryKey{equal}

	anyConsistency(client, defnID, equals)

	// query-consistency without any new mutations.
	ts := qclient.NewTsConsistency(vbnos, seqnos, vbuuids)
	queryConsistency(client, defnID, ts, equals, synch)
	<-synch

	// query-consistency with a new mutation.
	equal = common.SecondaryKey(querycmd.Arg2Key(constEqualLookup2))
	equals = []common.SecondaryKey{equal}
	seqno++
	ts = ts.Override(vbno, seqno, vbuuid)
	queryConsistency(client, defnID, ts, equals, synch)
	time.Sleep(2 * time.Second)
	setValueConst(b, maxvb, constDocValue2)
	<-synch

	// query-consistency with a new mutation.
	equal = common.SecondaryKey(querycmd.Arg2Key(constEqualLookup3))
	equals = []common.SecondaryKey{equal}
	seqno++
	ts = qclient.NewTsConsistency([]uint16{vbno}, []uint64{seqno}, []uint64{vbuuid})
	queryConsistency(client, defnID, ts, equals, synch)
	time.Sleep(2 * time.Second)
	setValueConst(b, maxvb, constDocValue3)
	<-synch

	// session-consistency without any new mutations.
	sessionConsistency(client, defnID, equals, synch)
	<-synch

	// session-consistency with a new mutation.
	setValueConst(b, maxvb, constDocValue4)
	equal = common.SecondaryKey(querycmd.Arg2Key(constEqualLookup4))
	equals = []common.SecondaryKey{equal}
	sessionConsistency(client, defnID, equals, synch)
	<-synch

	// session-consistency with a new mutation.
	equal = common.SecondaryKey(querycmd.Arg2Key(constEqualLookup5))
	equals = []common.SecondaryKey{equal}
	setValueConst(b, maxvb, constDocValue5)
	sessionConsistency(client, defnID, equals, synch)
	<-synch
	return nil
}
Beispiel #10
0
func startFeed(cluster, name string) {
	bucket, err := c.ConnectBucket(cluster, "default", "default")
	if err != nil {
		log.Fatal(err)
	}
	defer bucket.Close()
	options.maxVbs, err = c.MaxVbuckets(bucket)
	if err != nil {
		log.Fatal(err)
	}
	options.vbuckets = make([]uint16, 0, options.maxVbs)
	for i := 0; i < options.maxVbs; i++ {
		options.vbuckets = append(options.vbuckets, uint16(i))
	}

	// get dcp feed for this bucket.
	config := map[string]interface{}{
		"genChanSize":  10000,
		"dataChanSize": 10000,
	}
	dcpFeed, err := bucket.StartDcpFeed(name, uint32(0), 0xABCD, config)
	if err != nil {
		log.Fatal(err)
	}
	go func() {
		// start vbucket streams
		for _, vbno := range options.vbuckets {
			flags, vbuuid := uint32(0), uint64(0)
			start, end := uint64(0), uint64(0xFFFFFFFFFFFFFFFF)
			snapStart, snapEnd := uint64(0), uint64(0)
			err := dcpFeed.DcpRequestStream(
				vbno, vbno /*opaque*/, flags, vbuuid, start, end,
				snapStart, snapEnd)
			if err != nil {
				log.Fatal(err)
			}
			// FIXME/TODO: the below sleep avoid back-to-back dispatch of
			// StreamRequest to DCP, which seem to cause some problems.
			time.Sleep(1 * time.Millisecond)
		}
	}()
	tick := time.Tick(time.Second)
	countEvents := 0
	commands := make(map[byte]int)
	for {
		select {
		case e, ok := <-dcpFeed.C:
			if !ok {
				log.Fatal("dcpFeed channel has closed")
			}
			if _, ok := commands[byte(e.Opcode)]; !ok {
				commands[byte(e.Opcode)] = 0
			}
			commands[byte(e.Opcode)]++
			countEvents++

		case <-tick:
			log.Println("events received countEvents", countEvents)
			log.Println("commands received", commands)
		}
	}
}