func CountRange(indexName, bucketName, server string, low, high []interface{}, inclusion uint32,
consistency c.Consistency, vector *qc.TsConsistency) (int64, error) {
// ToDo: Create a client pool
client, e := CreateClient(server, "2itest")
if e != nil {
return 0, e
}
defer client.Close()
defnID, _ := GetDefnID(client, bucketName, indexName)
count, err := client.CountRange(defnID, c.SecondaryKey(low), c.SecondaryKey(high), qc.Inclusion(inclusion), consistency, vector)
if err != nil {
return 0, err
} else {
return count, nil
}
}
func RangeWithClient(indexName, bucketName, server string, low, high []interface{}, inclusion uint32,
distinct bool, limit int64, consistency c.Consistency, vector *qc.TsConsistency, client *qc.GsiClient) (tc.ScanResponse, error) {
var scanErr error
scanErr = nil
defnID, _ := GetDefnID(client, bucketName, indexName)
scanResults := make(tc.ScanResponse)
start := time.Now()
connErr := client.Range(
defnID, c.SecondaryKey(low), c.SecondaryKey(high), qc.Inclusion(inclusion), distinct, limit,
consistency, vector,
func(response qc.ResponseReader) bool {
if err := response.Error(); err != nil {
scanErr = err
return false
} else if skeys, pkeys, err := response.GetEntries(); err != nil {
scanErr = err
return false
} else {
for i, skey := range skeys {
primaryKey := string(pkeys[i])
if _, keyPresent := scanResults[primaryKey]; keyPresent {
// Duplicate primary key found
tc.HandleError(err, "Duplicate primary key found in the scan results: "+primaryKey)
} else {
scanResults[primaryKey] = skey
}
}
return true
}
return false
})
elapsed := time.Since(start)
if connErr != nil {
log.Printf("Connection error in Scan occured: %v", connErr)
return scanResults, connErr
} else if scanErr != nil {
return scanResults, scanErr
}
tc.LogPerfStat("Range", elapsed)
return scanResults, nil
}
func RunJob(client *qclient.GsiClient, job *Job, aggrQ chan *JobResult) {
var err error
var rows int64
spec := job.spec
result := job.result
if result != nil {
result.Id = spec.Id
}
errFn := func(e string) {
fmt.Printf("REQ:%d scan error occured: %s\n", spec.Id, e)
if result != nil {
platform.AddUint64(&result.ErrorCount, 1)
}
}
callb := func(res qclient.ResponseReader) bool {
if res.Error() != nil {
errFn(res.Error().Error())
return false
} else {
_, pkeys, err := res.GetEntries()
if err != nil {
errFn(err.Error())
return false
}
rows += int64(len(pkeys))
}
return true
}
var cons c.Consistency
if spec.Consistency {
cons = c.SessionConsistency
} else {
cons = c.AnyConsistency
}
startTime := time.Now()
switch spec.Type {
case "All":
err = client.ScanAll(spec.DefnId, spec.Limit, cons, nil, callb)
case "Range":
err = client.Range(spec.DefnId, spec.Low, spec.High,
qclient.Inclusion(spec.Inclusion), false, spec.Limit, cons, nil, callb)
case "Lookup":
err = client.Lookup(spec.DefnId, spec.Lookups, false,
spec.Limit, cons, nil, callb)
}
if err != nil {
errFn(err.Error())
}
dur := time.Now().Sub(startTime)
if result != nil {
aggrQ <- &JobResult{
job: job,
dur: dur.Nanoseconds(),
rows: rows,
}
}
}
// ParseArgs into Command object, return the list of arguments,
// flagset used for parseing and error if any.
func ParseArgs(arguments []string) (*Command, []string, *flag.FlagSet, error) {
var fields, bindexes string
var inclusion uint
var equal, low, high string
var useSessionCons bool
cmdOptions := &Command{Consistency: c.AnyConsistency}
fset := flag.NewFlagSet("cmd", flag.ExitOnError)
// basic options
fset.StringVar(&cmdOptions.Server, "server", "", "Cluster server address")
fset.StringVar(&cmdOptions.Auth, "auth", "", "Auth user and password")
fset.StringVar(&cmdOptions.Bucket, "bucket", "", "Bucket name")
fset.StringVar(&cmdOptions.OpType, "type", "", "Command: scan|stats|scanAll|count|nodes|create|build|drop|list|config")
fset.StringVar(&cmdOptions.IndexName, "index", "", "Index name")
// options for create-index
fset.StringVar(&cmdOptions.WhereStr, "where", "", "where clause for create index")
fset.StringVar(&fields, "fields", "", "Comma separated on-index fields") // secStrs
fset.BoolVar(&cmdOptions.IsPrimary, "primary", false, "Is primary index")
fset.StringVar(&cmdOptions.With, "with", "", "index specific properties")
// options for build-indexes, drop-indexes
fset.StringVar(&bindexes, "indexes", "", "csv list of bucket.index to build")
// options for Range, Statistics, Count
fset.StringVar(&low, "low", "[]", "Span.Range: [low]")
fset.StringVar(&high, "high", "[]", "Span.Range: [high]")
fset.StringVar(&equal, "equal", "", "Span.Lookup: [key]")
fset.UintVar(&inclusion, "incl", 0, "Range: 0|1|2|3")
fset.Int64Var(&cmdOptions.Limit, "limit", 10, "Row limit")
fset.BoolVar(&cmdOptions.Help, "h", false, "print help")
fset.BoolVar(&useSessionCons, "consistency", false, "Use session consistency")
// options for setting configuration
fset.StringVar(&cmdOptions.ConfigKey, "ckey", "", "Config key")
fset.StringVar(&cmdOptions.ConfigVal, "cval", "", "Config value")
fset.StringVar(&cmdOptions.Using, "using", c.ForestDB, "storate type to use")
// not useful to expose in sherlock
cmdOptions.ExprType = "N1QL"
cmdOptions.PartnStr = "partn"
if err := fset.Parse(arguments); err != nil {
return nil, nil, fset, err
}
if useSessionCons {
cmdOptions.Consistency = c.SessionConsistency
}
// if server is not specified, try guessing
if cmdOptions.Server == "" {
if guess := guessServer(); guess != "" {
cmdOptions.Server = guess
fset.Set("server", guess)
}
}
// if server is not specified, try guessing
if cmdOptions.Auth == "" {
if guess := guessAuth(cmdOptions.Server); guess != "" {
cmdOptions.Auth = guess
fset.Set("auth", guess)
}
}
// validate combinations
err := validate(cmdOptions, fset)
if err != nil {
return nil, nil, fset, err
}
// bindexes
if len(bindexes) > 0 {
cmdOptions.Bindexes = strings.Split(bindexes, ",")
}
// inclusion, secStrs, equal, low, high
cmdOptions.Inclusion = qclient.Inclusion(inclusion)
cmdOptions.SecStrs = make([]string, 0)
if fields != "" {
for _, field := range strings.Split(fields, ",") {
expr, err := n1ql.ParseExpression(field)
if err != nil {
msgf := "Error occured: Invalid field (%v) %v\n"
return nil, nil, fset, fmt.Errorf(msgf, field, err)
}
secStr := expression.NewStringer().Visit(expr)
cmdOptions.SecStrs = append(cmdOptions.SecStrs, secStr)
}
}
if equal != "" {
cmdOptions.Equal = c.SecondaryKey(Arg2Key([]byte(equal)))
}
cmdOptions.Low = c.SecondaryKey(Arg2Key([]byte(low)))
cmdOptions.High = c.SecondaryKey(Arg2Key([]byte(high)))
// with
if len(cmdOptions.With) > 0 {
err := json.Unmarshal([]byte(cmdOptions.With), &cmdOptions.WithPlan)
if err != nil {
logging.Fatalf("%v\n", err)
os.Exit(1)
}
}
// setup cbauth
if cmdOptions.Auth != "" {
up := strings.Split(cmdOptions.Auth, ":")
_, err := cbauth.InternalRetryDefaultInit(cmdOptions.Server, up[0], up[1])
if err != nil {
logging.Fatalf("Failed to initialize cbauth: %s\n", err)
os.Exit(1)
}
}
return cmdOptions, fset.Args(), fset, err
}
func Range(indexName, bucketName, server string, low, high []interface{}, inclusion uint32,
distinct bool, limit int64, consistency c.Consistency, vector *qc.TsConsistency) (tc.ScanResponse, error) {
var scanErr error
scanErr = nil
var previousSecKey value.Value
// ToDo: Create a client pool
client, e := CreateClient(server, "2itest")
if e != nil {
return nil, e
}
defer client.Close()
defnID, _ := GetDefnID(client, bucketName, indexName)
scanResults := make(tc.ScanResponse)
start := time.Now()
connErr := client.Range(
defnID, c.SecondaryKey(low), c.SecondaryKey(high), qc.Inclusion(inclusion), distinct, limit,
consistency, vector,
func(response qc.ResponseReader) bool {
if err := response.Error(); err != nil {
scanErr = err
return false
} else if skeys, pkeys, err := response.GetEntries(); err != nil {
scanErr = err
return false
} else {
for i, skey := range skeys {
primaryKey := string(pkeys[i])
if _, keyPresent := scanResults[primaryKey]; keyPresent {
// Duplicate primary key found
tc.HandleError(err, "Duplicate primary key found in the scan results: "+primaryKey)
} else {
// Test collation only if CheckCollation is true
if CheckCollation == true && len(skey) > 0 {
secVal := skey2Values(skey)[0]
if previousSecKey == nil {
previousSecKey = secVal
} else {
if secVal.Collate(previousSecKey) < 0 {
errMsg := "Collation check failed. Previous Sec key > Current Sec key"
scanErr = errors.New(errMsg)
return false
}
}
}
scanResults[primaryKey] = skey
}
}
return true
}
return false
})
elapsed := time.Since(start)
if connErr != nil {
log.Printf("Connection error in Scan occured: %v", connErr)
return scanResults, connErr
} else if scanErr != nil {
return scanResults, scanErr
}
tc.LogPerfStat("Range", elapsed)
return scanResults, nil
}
func RangeStatistics(indexName, bucketName, server string, low, high []interface{}, inclusion uint32) error {
// ToDo: Create a client pool
client, e := CreateClient(server, "2itest")
if e != nil {
return e
}
defer client.Close()
defnID, _ := GetDefnID(client, bucketName, indexName)
statistics, err := client.RangeStatistics(defnID, c.SecondaryKey(low), c.SecondaryKey(high), qc.Inclusion(inclusion))
if err != nil {
return err
} else {
log.Print("Statistics: %v\n\n", statistics)
return nil
}
}