Example #1
0
func (m *mcops) count(i, n int, err error) {
	if n < 2 {
		// Too short to actually know the opcode
		i = 256
	}
	if err == nil {
		platform.AddUint64(&m.success[i], 1)
	} else {
		platform.AddUint64(&m.errored[i], 1)
	}
	platform.AddUint64(&m.moved[i], uint64(n))
}
func (fdb *fdbSlice) logWriterStat() {
	count := platform.AddUint64(&fdb.flushedCount, 1)
	if (count%10000 == 0) || count == 1 {
		logging.Infof("logWriterStat:: %v "+
			"FlushedCount %v QueuedCount %v", fdb.idxInstId,
			count, len(fdb.cmdCh))
	}

}
Example #3
0
func (s *scanCoordinator) newRequest(protoReq interface{},
	cancelCh <-chan interface{}) (r *ScanRequest, err error) {

	var indexInst *common.IndexInst
	r = new(ScanRequest)
	r.ScanId = platform.AddUint64(&s.reqCounter, 1)
	r.LogPrefix = fmt.Sprintf("SCAN##%d", r.ScanId)

	cfg := s.config.Load()
	timeout := time.Millisecond * time.Duration(cfg["settings.scan_timeout"].Int())

	if timeout != 0 {
		r.ExpiredTime = time.Now().Add(timeout)
		r.TimeoutCh = time.After(timeout)
	}

	r.CancelCh = cancelCh

	isNil := func(k []byte) bool {
		if len(k) == 0 || (!r.isPrimary && string(k) == "[]") {
			return true
		}
		return false
	}

	newKey := func(k []byte) (IndexKey, error) {
		if len(k) == 0 {
			return nil, fmt.Errorf("Key is null")
		}

		if r.isPrimary {
			return NewPrimaryKey(k)
		} else {
			return NewSecondaryKey(k)
		}
	}

	newLowKey := func(k []byte) (IndexKey, error) {
		if isNil(k) {
			return MinIndexKey, nil
		}

		return newKey(k)
	}

	newHighKey := func(k []byte) (IndexKey, error) {
		if isNil(k) {
			return MaxIndexKey, nil
		}

		return newKey(k)
	}

	fillRanges := func(low, high []byte, keys [][]byte) {
		var key IndexKey
		var localErr error
		defer func() {
			if err == nil {
				err = localErr
			}
		}()

		// range
		r.LowBytes = low
		r.HighBytes = high

		if r.Low, localErr = newLowKey(low); localErr != nil {
			localErr = fmt.Errorf("Invalid low key %s (%s)", string(low), localErr)
			return
		}

		if r.High, localErr = newHighKey(high); localErr != nil {
			localErr = fmt.Errorf("Invalid high key %s (%s)", string(high), localErr)
			return
		}

		// point query for keys
		for _, k := range keys {
			r.KeysBytes = append(r.KeysBytes, k)
			if key, localErr = newKey(k); localErr != nil {
				localErr = fmt.Errorf("Invalid equal key %s (%s)", string(k), localErr)
				return
			}
			r.Keys = append(r.Keys, key)
		}
	}

	setConsistency := func(
		cons common.Consistency, vector *protobuf.TsConsistency) {

		r.Consistency = &cons
		cfg := s.config.Load()
		if cons == common.QueryConsistency && vector != nil {
			r.Ts = common.NewTsVbuuid("", cfg["numVbuckets"].Int())
			// if vector == nil, it is similar to AnyConsistency
			for i, vbno := range vector.Vbnos {
				r.Ts.Seqnos[vbno] = vector.Seqnos[i]
				r.Ts.Vbuuids[vbno] = vector.Vbuuids[i]
			}

		} else if cons == common.SessionConsistency {
			r.Ts = common.NewTsVbuuid("", cfg["numVbuckets"].Int())
			r.Ts.Seqnos = vector.Seqnos // full set of seqnos.
			r.Ts.Crc64 = vector.GetCrc64()
		}
	}

	setIndexParams := func() {
		var localErr error
		defer func() {
			if err == nil {
				err = localErr
			}
		}()
		s.mu.RLock()
		defer s.mu.RUnlock()

		stats := s.stats.Get()
		indexInst, localErr = s.findIndexInstance(r.DefnID)
		if localErr == nil {
			r.isPrimary = indexInst.Defn.IsPrimary
			r.IndexName, r.Bucket = indexInst.Defn.Name, indexInst.Defn.Bucket
			r.IndexInstId = indexInst.InstId
			if r.Ts != nil {
				r.Ts.Bucket = r.Bucket
			}
			if indexInst.State != common.INDEX_STATE_ACTIVE {
				localErr = ErrIndexNotReady
			} else {
				r.Stats = stats.indexes[r.IndexInstId]
			}
		}
	}

	switch req := protoReq.(type) {
	case *protobuf.StatisticsRequest:
		r.DefnID = req.GetDefnID()
		r.ScanType = StatsReq
		r.Incl = Inclusion(req.GetSpan().GetRange().GetInclusion())
		setIndexParams()
		fillRanges(
			req.GetSpan().GetRange().GetLow(),
			req.GetSpan().GetRange().GetHigh(),
			req.GetSpan().GetEquals())

	case *protobuf.CountRequest:
		r.DefnID = req.GetDefnID()
		cons := common.Consistency(req.GetCons())
		vector := req.GetVector()
		setConsistency(cons, vector)
		r.ScanType = CountReq
		r.Incl = Inclusion(req.GetSpan().GetRange().GetInclusion())
		setIndexParams()
		fillRanges(
			req.GetSpan().GetRange().GetLow(),
			req.GetSpan().GetRange().GetHigh(),
			req.GetSpan().GetEquals())

	case *protobuf.ScanRequest:
		r.DefnID = req.GetDefnID()
		cons := common.Consistency(req.GetCons())
		vector := req.GetVector()
		setConsistency(cons, vector)
		r.ScanType = ScanReq
		r.Incl = Inclusion(req.GetSpan().GetRange().GetInclusion())
		setIndexParams()
		fillRanges(
			req.GetSpan().GetRange().GetLow(),
			req.GetSpan().GetRange().GetHigh(),
			req.GetSpan().GetEquals())
		r.Limit = req.GetLimit()
	case *protobuf.ScanAllRequest:
		r.DefnID = req.GetDefnID()
		cons := common.Consistency(req.GetCons())
		vector := req.GetVector()
		setConsistency(cons, vector)
		r.ScanType = ScanAllReq
		r.Limit = req.GetLimit()
		setIndexParams()
	default:
		err = ErrUnsupportedRequest
	}

	return
}
Example #4
0
func RunJob(client *qclient.GsiClient, job *Job, aggrQ chan *JobResult) {
	var err error
	var rows int64

	spec := job.spec
	result := job.result
	if result != nil {
		result.Id = spec.Id
	}

	errFn := func(e string) {
		fmt.Printf("REQ:%d scan error occured: %s\n", spec.Id, e)
		if result != nil {
			platform.AddUint64(&result.ErrorCount, 1)
		}
	}

	callb := func(res qclient.ResponseReader) bool {
		if res.Error() != nil {
			errFn(res.Error().Error())
			return false
		} else {
			_, pkeys, err := res.GetEntries()
			if err != nil {
				errFn(err.Error())
				return false
			}

			rows += int64(len(pkeys))
		}

		return true
	}

	var cons c.Consistency
	if spec.Consistency {
		cons = c.SessionConsistency
	} else {
		cons = c.AnyConsistency
	}

	startTime := time.Now()
	switch spec.Type {
	case "All":
		err = client.ScanAll(spec.DefnId, spec.Limit, cons, nil, callb)
	case "Range":
		err = client.Range(spec.DefnId, spec.Low, spec.High,
			qclient.Inclusion(spec.Inclusion), false, spec.Limit, cons, nil, callb)
	case "Lookup":
		err = client.Lookup(spec.DefnId, spec.Lookups, false,
			spec.Limit, cons, nil, callb)
	}

	if err != nil {
		errFn(err.Error())
	}

	dur := time.Now().Sub(startTime)

	if result != nil {
		aggrQ <- &JobResult{
			job:  job,
			dur:  dur.Nanoseconds(),
			rows: rows,
		}
	}
}