示例#1
0
func CountRange(indexName, bucketName, server string, low, high []interface{}, inclusion uint32,
	consistency c.Consistency, vector *qc.TsConsistency) (int64, error) {
	// ToDo: Create a client pool
	client, e := CreateClient(server, "2itest")
	if e != nil {
		return 0, e
	}
	defer client.Close()

	defnID, _ := GetDefnID(client, bucketName, indexName)
	count, err := client.CountRange(defnID, c.SecondaryKey(low), c.SecondaryKey(high), qc.Inclusion(inclusion), consistency, vector)
	if err != nil {
		return 0, err
	} else {
		return count, nil
	}
}
示例#2
0
func RangeWithClient(indexName, bucketName, server string, low, high []interface{}, inclusion uint32,
	distinct bool, limit int64, consistency c.Consistency, vector *qc.TsConsistency, client *qc.GsiClient) (tc.ScanResponse, error) {
	var scanErr error
	scanErr = nil
	defnID, _ := GetDefnID(client, bucketName, indexName)
	scanResults := make(tc.ScanResponse)

	start := time.Now()
	connErr := client.Range(
		defnID, c.SecondaryKey(low), c.SecondaryKey(high), qc.Inclusion(inclusion), distinct, limit,
		consistency, vector,
		func(response qc.ResponseReader) bool {
			if err := response.Error(); err != nil {
				scanErr = err
				return false
			} else if skeys, pkeys, err := response.GetEntries(); err != nil {
				scanErr = err
				return false
			} else {
				for i, skey := range skeys {
					primaryKey := string(pkeys[i])
					if _, keyPresent := scanResults[primaryKey]; keyPresent {
						// Duplicate primary key found
						tc.HandleError(err, "Duplicate primary key found in the scan results: "+primaryKey)
					} else {
						scanResults[primaryKey] = skey
					}
				}
				return true
			}
			return false
		})
	elapsed := time.Since(start)

	if connErr != nil {
		log.Printf("Connection error in Scan occured: %v", connErr)
		return scanResults, connErr
	} else if scanErr != nil {
		return scanResults, scanErr
	}

	tc.LogPerfStat("Range", elapsed)
	return scanResults, nil
}
示例#3
0
func RunJob(client *qclient.GsiClient, job *Job, aggrQ chan *JobResult) {
	var err error
	var rows int64

	spec := job.spec
	result := job.result
	if result != nil {
		result.Id = spec.Id
	}

	errFn := func(e string) {
		fmt.Printf("REQ:%d scan error occured: %s\n", spec.Id, e)
		if result != nil {
			platform.AddUint64(&result.ErrorCount, 1)
		}
	}

	callb := func(res qclient.ResponseReader) bool {
		if res.Error() != nil {
			errFn(res.Error().Error())
			return false
		} else {
			_, pkeys, err := res.GetEntries()
			if err != nil {
				errFn(err.Error())
				return false
			}

			rows += int64(len(pkeys))
		}

		return true
	}

	var cons c.Consistency
	if spec.Consistency {
		cons = c.SessionConsistency
	} else {
		cons = c.AnyConsistency
	}

	startTime := time.Now()
	switch spec.Type {
	case "All":
		err = client.ScanAll(spec.DefnId, spec.Limit, cons, nil, callb)
	case "Range":
		err = client.Range(spec.DefnId, spec.Low, spec.High,
			qclient.Inclusion(spec.Inclusion), false, spec.Limit, cons, nil, callb)
	case "Lookup":
		err = client.Lookup(spec.DefnId, spec.Lookups, false,
			spec.Limit, cons, nil, callb)
	}

	if err != nil {
		errFn(err.Error())
	}

	dur := time.Now().Sub(startTime)

	if result != nil {
		aggrQ <- &JobResult{
			job:  job,
			dur:  dur.Nanoseconds(),
			rows: rows,
		}
	}
}
示例#4
0
文件: docmd.go 项目: jchris/indexing
// ParseArgs into Command object, return the list of arguments,
// flagset used for parseing and error if any.
func ParseArgs(arguments []string) (*Command, []string, *flag.FlagSet, error) {
	var fields, bindexes string
	var inclusion uint
	var equal, low, high string
	var useSessionCons bool

	cmdOptions := &Command{Consistency: c.AnyConsistency}
	fset := flag.NewFlagSet("cmd", flag.ExitOnError)

	// basic options
	fset.StringVar(&cmdOptions.Server, "server", "", "Cluster server address")
	fset.StringVar(&cmdOptions.Auth, "auth", "", "Auth user and password")
	fset.StringVar(&cmdOptions.Bucket, "bucket", "", "Bucket name")
	fset.StringVar(&cmdOptions.OpType, "type", "", "Command: scan|stats|scanAll|count|nodes|create|build|drop|list|config")
	fset.StringVar(&cmdOptions.IndexName, "index", "", "Index name")
	// options for create-index
	fset.StringVar(&cmdOptions.WhereStr, "where", "", "where clause for create index")
	fset.StringVar(&fields, "fields", "", "Comma separated on-index fields") // secStrs
	fset.BoolVar(&cmdOptions.IsPrimary, "primary", false, "Is primary index")
	fset.StringVar(&cmdOptions.With, "with", "", "index specific properties")
	// options for build-indexes, drop-indexes
	fset.StringVar(&bindexes, "indexes", "", "csv list of bucket.index to build")
	// options for Range, Statistics, Count
	fset.StringVar(&low, "low", "[]", "Span.Range: [low]")
	fset.StringVar(&high, "high", "[]", "Span.Range: [high]")
	fset.StringVar(&equal, "equal", "", "Span.Lookup: [key]")
	fset.UintVar(&inclusion, "incl", 0, "Range: 0|1|2|3")
	fset.Int64Var(&cmdOptions.Limit, "limit", 10, "Row limit")
	fset.BoolVar(&cmdOptions.Help, "h", false, "print help")
	fset.BoolVar(&useSessionCons, "consistency", false, "Use session consistency")
	// options for setting configuration
	fset.StringVar(&cmdOptions.ConfigKey, "ckey", "", "Config key")
	fset.StringVar(&cmdOptions.ConfigVal, "cval", "", "Config value")
	fset.StringVar(&cmdOptions.Using, "using", c.ForestDB, "storate type to use")

	// not useful to expose in sherlock
	cmdOptions.ExprType = "N1QL"
	cmdOptions.PartnStr = "partn"

	if err := fset.Parse(arguments); err != nil {
		return nil, nil, fset, err
	}

	if useSessionCons {
		cmdOptions.Consistency = c.SessionConsistency
	}

	// if server is not specified, try guessing
	if cmdOptions.Server == "" {
		if guess := guessServer(); guess != "" {
			cmdOptions.Server = guess
			fset.Set("server", guess)
		}
	}

	// if server is not specified, try guessing
	if cmdOptions.Auth == "" {
		if guess := guessAuth(cmdOptions.Server); guess != "" {
			cmdOptions.Auth = guess
			fset.Set("auth", guess)
		}
	}

	// validate combinations
	err := validate(cmdOptions, fset)
	if err != nil {
		return nil, nil, fset, err
	}
	// bindexes
	if len(bindexes) > 0 {
		cmdOptions.Bindexes = strings.Split(bindexes, ",")
	}

	// inclusion, secStrs, equal, low, high
	cmdOptions.Inclusion = qclient.Inclusion(inclusion)
	cmdOptions.SecStrs = make([]string, 0)
	if fields != "" {
		for _, field := range strings.Split(fields, ",") {
			expr, err := n1ql.ParseExpression(field)
			if err != nil {
				msgf := "Error occured: Invalid field (%v) %v\n"
				return nil, nil, fset, fmt.Errorf(msgf, field, err)
			}
			secStr := expression.NewStringer().Visit(expr)
			cmdOptions.SecStrs = append(cmdOptions.SecStrs, secStr)
		}
	}
	if equal != "" {
		cmdOptions.Equal = c.SecondaryKey(Arg2Key([]byte(equal)))
	}
	cmdOptions.Low = c.SecondaryKey(Arg2Key([]byte(low)))
	cmdOptions.High = c.SecondaryKey(Arg2Key([]byte(high)))

	// with
	if len(cmdOptions.With) > 0 {
		err := json.Unmarshal([]byte(cmdOptions.With), &cmdOptions.WithPlan)
		if err != nil {
			logging.Fatalf("%v\n", err)
			os.Exit(1)
		}
	}

	// setup cbauth
	if cmdOptions.Auth != "" {
		up := strings.Split(cmdOptions.Auth, ":")
		_, err := cbauth.InternalRetryDefaultInit(cmdOptions.Server, up[0], up[1])
		if err != nil {
			logging.Fatalf("Failed to initialize cbauth: %s\n", err)
			os.Exit(1)
		}
	}

	return cmdOptions, fset.Args(), fset, err
}
示例#5
0
func Range(indexName, bucketName, server string, low, high []interface{}, inclusion uint32,
	distinct bool, limit int64, consistency c.Consistency, vector *qc.TsConsistency) (tc.ScanResponse, error) {
	var scanErr error
	scanErr = nil
	var previousSecKey value.Value

	// ToDo: Create a client pool
	client, e := CreateClient(server, "2itest")
	if e != nil {
		return nil, e
	}
	defer client.Close()

	defnID, _ := GetDefnID(client, bucketName, indexName)
	scanResults := make(tc.ScanResponse)

	start := time.Now()
	connErr := client.Range(
		defnID, c.SecondaryKey(low), c.SecondaryKey(high), qc.Inclusion(inclusion), distinct, limit,
		consistency, vector,
		func(response qc.ResponseReader) bool {
			if err := response.Error(); err != nil {
				scanErr = err
				return false
			} else if skeys, pkeys, err := response.GetEntries(); err != nil {
				scanErr = err
				return false
			} else {
				for i, skey := range skeys {
					primaryKey := string(pkeys[i])
					if _, keyPresent := scanResults[primaryKey]; keyPresent {
						// Duplicate primary key found
						tc.HandleError(err, "Duplicate primary key found in the scan results: "+primaryKey)
					} else {
						// Test collation only if CheckCollation is true
						if CheckCollation == true && len(skey) > 0 {
							secVal := skey2Values(skey)[0]
							if previousSecKey == nil {
								previousSecKey = secVal
							} else {
								if secVal.Collate(previousSecKey) < 0 {
									errMsg := "Collation check failed. Previous Sec key > Current Sec key"
									scanErr = errors.New(errMsg)
									return false
								}
							}
						}

						scanResults[primaryKey] = skey
					}
				}
				return true
			}
			return false
		})
	elapsed := time.Since(start)

	if connErr != nil {
		log.Printf("Connection error in Scan occured: %v", connErr)
		return scanResults, connErr
	} else if scanErr != nil {
		return scanResults, scanErr
	}

	tc.LogPerfStat("Range", elapsed)
	return scanResults, nil
}
示例#6
0
func RangeStatistics(indexName, bucketName, server string, low, high []interface{}, inclusion uint32) error {
	// ToDo: Create a client pool
	client, e := CreateClient(server, "2itest")
	if e != nil {
		return e
	}
	defer client.Close()

	defnID, _ := GetDefnID(client, bucketName, indexName)
	statistics, err := client.RangeStatistics(defnID, c.SecondaryKey(low), c.SecondaryKey(high), qc.Inclusion(inclusion))
	if err != nil {
		return err
	} else {
		log.Print("Statistics: %v\n\n", statistics)
		return nil
	}
}