func RangeWithClient(indexName, bucketName, server string, low, high []interface{}, inclusion uint32,
distinct bool, limit int64, consistency c.Consistency, vector *qc.TsConsistency, client *qc.GsiClient) (tc.ScanResponse, error) {
var scanErr error
scanErr = nil
defnID, _ := GetDefnID(client, bucketName, indexName)
scanResults := make(tc.ScanResponse)
start := time.Now()
connErr := client.Range(
defnID, c.SecondaryKey(low), c.SecondaryKey(high), qc.Inclusion(inclusion), distinct, limit,
consistency, vector,
func(response qc.ResponseReader) bool {
if err := response.Error(); err != nil {
scanErr = err
return false
} else if skeys, pkeys, err := response.GetEntries(); err != nil {
scanErr = err
return false
} else {
for i, skey := range skeys {
primaryKey := string(pkeys[i])
if _, keyPresent := scanResults[primaryKey]; keyPresent {
// Duplicate primary key found
tc.HandleError(err, "Duplicate primary key found in the scan results: "+primaryKey)
} else {
scanResults[primaryKey] = skey
}
}
return true
}
return false
})
elapsed := time.Since(start)
if connErr != nil {
log.Printf("Connection error in Scan occured: %v", connErr)
return scanResults, connErr
} else if scanErr != nil {
return scanResults, scanErr
}
tc.LogPerfStat("Range", elapsed)
return scanResults, nil
}
func RunJob(client *qclient.GsiClient, job *Job, aggrQ chan *JobResult) {
var err error
var rows int64
spec := job.spec
result := job.result
if result != nil {
result.Id = spec.Id
}
errFn := func(e string) {
fmt.Printf("REQ:%d scan error occured: %s\n", spec.Id, e)
if result != nil {
platform.AddUint64(&result.ErrorCount, 1)
}
}
callb := func(res qclient.ResponseReader) bool {
if res.Error() != nil {
errFn(res.Error().Error())
return false
} else {
_, pkeys, err := res.GetEntries()
if err != nil {
errFn(err.Error())
return false
}
rows += int64(len(pkeys))
}
return true
}
var cons c.Consistency
if spec.Consistency {
cons = c.SessionConsistency
} else {
cons = c.AnyConsistency
}
startTime := time.Now()
switch spec.Type {
case "All":
err = client.ScanAll(spec.DefnId, spec.Limit, cons, nil, callb)
case "Range":
err = client.Range(spec.DefnId, spec.Low, spec.High,
qclient.Inclusion(spec.Inclusion), false, spec.Limit, cons, nil, callb)
case "Lookup":
err = client.Lookup(spec.DefnId, spec.Lookups, false,
spec.Limit, cons, nil, callb)
}
if err != nil {
errFn(err.Error())
}
dur := time.Now().Sub(startTime)
if result != nil {
aggrQ <- &JobResult{
job: job,
dur: dur.Nanoseconds(),
rows: rows,
}
}
}
// HandleCommand after parsing it with ParseArgs().
func HandleCommand(
client *qclient.GsiClient,
cmd *Command,
verbose bool,
w io.Writer) (err error) {
iname, bucket, limit := cmd.IndexName, cmd.Bucket, cmd.Limit
low, high, equal, incl := cmd.Low, cmd.High, cmd.Equal, cmd.Inclusion
cons := cmd.Consistency
indexes, err := client.Refresh()
entries := 0
callb := func(res qclient.ResponseReader) bool {
if res.Error() != nil {
fmt.Fprintln(w, "Error: ", res)
} else if skeys, pkeys, err := res.GetEntries(); err != nil {
fmt.Fprintln(w, "Error: ", err)
} else {
if verbose == false {
for i, pkey := range pkeys {
fmt.Fprintf(w, "%v ... %v\n", skeys[i], string(pkey))
}
}
entries += len(pkeys)
}
return true
}
switch cmd.OpType {
case "nodes":
fmt.Fprintln(w, "List of nodes:")
nodes, err := client.Nodes()
if err != nil {
return err
}
for _, n := range nodes {
fmsg := " {%v, %v, %q}\n"
fmt.Fprintf(w, fmsg, n.Adminport, n.Queryport, n.Status)
}
case "list":
time.Sleep(2 * time.Second)
indexes, err = client.Refresh()
if err != nil {
return err
}
fmt.Fprintln(w, "List of indexes:")
for _, index := range indexes {
printIndexInfo(w, index)
}
case "create":
var defnID uint64
if len(cmd.SecStrs) == 0 && !cmd.IsPrimary || cmd.IndexName == "" {
return fmt.Errorf("createIndex(): required fields missing")
}
defnID, err = client.CreateIndex(
iname, bucket, cmd.Using, cmd.ExprType,
cmd.PartnStr, cmd.WhereStr, cmd.SecStrs, cmd.IsPrimary,
[]byte(cmd.With))
if err == nil {
fmt.Fprintf(w, "Index created: %v with %q\n", defnID, cmd.With)
}
case "build":
defnIDs := make([]uint64, 0, len(cmd.Bindexes))
for _, bindex := range cmd.Bindexes {
v := strings.Split(bindex, ":")
if len(v) < 0 {
return fmt.Errorf("invalid index specified : %v", bindex)
}
bucket, iname = v[0], v[1]
index, ok := GetIndex(client, bucket, iname)
if ok {
defnIDs = append(defnIDs, uint64(index.Definition.DefnId))
} else {
err = fmt.Errorf("index %v/%v unknown", bucket, iname)
break
}
}
if err == nil {
err = client.BuildIndexes(defnIDs)
fmt.Fprintf(w, "Index building for: %v\n", defnIDs)
}
case "drop":
index, ok := GetIndex(client, cmd.Bucket, cmd.IndexName)
if !ok {
return fmt.Errorf("invalid index specified : %v", cmd.IndexName)
}
err = client.DropIndex(uint64(index.Definition.DefnId))
if err == nil {
fmt.Fprintf(w, "Index dropped %v/%v\n", bucket, iname)
} else {
err = fmt.Errorf("index %v/%v drop failed", bucket, iname)
break
}
case "scan":
var state c.IndexState
index, _ := GetIndex(client, bucket, iname)
defnID := uint64(index.Definition.DefnId)
fmt.Fprintln(w, "Scan index:")
_, err = WaitUntilIndexState(
client, []uint64{defnID}, c.INDEX_STATE_ACTIVE,
100 /*period*/, 20000 /*timeout*/)
if err != nil {
state, err = client.IndexState(defnID)
fmt.Fprintf(w, "Index state: {%v, %v}\n", state, err)
} else if cmd.Equal != nil {
equals := []c.SecondaryKey{cmd.Equal}
client.Lookup(
uint64(defnID), equals, false, limit,
cons, nil, callb)
} else {
err = client.Range(
uint64(defnID), low, high, incl, false, limit,
cons, nil, callb)
}
if err == nil {
fmt.Fprintln(w, "Total number of entries: ", entries)
}
case "scanAll":
var state c.IndexState
index, found := GetIndex(client, bucket, iname)
if !found {
fmt.Fprintln(w, "Index not found")
os.Exit(1)
}
defnID := uint64(index.Definition.DefnId)
fmt.Fprintln(w, "ScanAll index:")
_, err = WaitUntilIndexState(
client, []uint64{defnID}, c.INDEX_STATE_ACTIVE,
100 /*period*/, 20000 /*timeout*/)
if err != nil {
state, err = client.IndexState(defnID)
fmt.Fprintf(w, "Index state: {%v, %v} \n", state, err)
} else {
err = client.ScanAll(
uint64(defnID), limit, cons, nil, callb)
}
if err == nil {
fmt.Fprintln(w, "Total number of entries: ", entries)
}
case "stats":
var state c.IndexState
var statsResp c.IndexStatistics
index, _ := GetIndex(client, bucket, iname)
defnID := uint64(index.Definition.DefnId)
_, err = WaitUntilIndexState(
client, []uint64{defnID}, c.INDEX_STATE_ACTIVE,
100 /*period*/, 20000 /*timeout*/)
if err != nil {
state, err = client.IndexState(defnID)
fmt.Fprintf(w, "Index state: {%v, %v} \n", state, err)
} else if cmd.Equal != nil {
statsResp, err = client.LookupStatistics(uint64(defnID), equal)
} else {
statsResp, err = client.RangeStatistics(
uint64(defnID), low, high, incl)
}
if err == nil {
fmt.Fprintln(w, "Stats: ", statsResp)
}
case "count":
var state c.IndexState
var count int64
index, _ := GetIndex(client, bucket, iname)
defnID := uint64(index.Definition.DefnId)
_, err = WaitUntilIndexState(
client, []uint64{defnID}, c.INDEX_STATE_ACTIVE,
100 /*period*/, 20000 /*timeout*/)
if err != nil {
state, err = client.IndexState(defnID)
fmt.Fprintf(w, "Index state: {%v, %v} \n", state, err)
} else if cmd.Equal != nil {
fmt.Fprintln(w, "CountLookup:")
equals := []c.SecondaryKey{cmd.Equal}
count, err := client.CountLookup(uint64(defnID), equals, cons, nil)
if err == nil {
fmt.Fprintf(w, "Index %q/%q has %v entries\n", bucket, iname, count)
}
} else {
fmt.Fprintln(w, "CountRange:")
count, err = client.CountRange(uint64(defnID), low, high, incl, cons, nil)
if err == nil {
fmt.Fprintf(w, "Index %q/%q has %v entries\n", bucket, iname, count)
}
}
case "config":
nodes, err := client.Nodes()
if err != nil {
return err
}
var adminurl string
for _, indexer := range nodes {
adminurl = indexer.Adminport
break
}
host, sport, _ := net.SplitHostPort(adminurl)
iport, _ := strconv.Atoi(sport)
client := http.Client{}
//
// hack, fix this
//
ihttp := iport + 2
url := "http://" + host + ":" + strconv.Itoa(ihttp) + "/settings"
oreq, err := http.NewRequest("GET", url, nil)
if cmd.Auth != "" {
up := strings.Split(cmd.Auth, ":")
oreq.SetBasicAuth(up[0], up[1])
}
oresp, err := client.Do(oreq)
if err != nil {
return err
}
obody, err := ioutil.ReadAll(oresp.Body)
if err != nil {
return err
}
oresp.Body.Close()
pretty := strings.Replace(string(obody), ",\"", ",\n\"", -1)
fmt.Printf("Current Settings:\n%s\n", string(pretty))
var jbody map[string]interface{}
err = json.Unmarshal(obody, &jbody)
if err != nil {
return err
}
if len(cmd.ConfigKey) > 0 {
fmt.Printf("Changing config key '%s' to value '%s'\n", cmd.ConfigKey, cmd.ConfigVal)
jbody[cmd.ConfigKey] = cmd.ConfigVal
pbody, err := json.Marshal(jbody)
if err != nil {
return err
}
preq, err := http.NewRequest("POST", url, bytes.NewBuffer(pbody))
if cmd.Auth != "" {
up := strings.Split(cmd.Auth, ":")
preq.SetBasicAuth(up[0], up[1])
}
_, err = client.Do(preq)
if err != nil {
return err
}
nresp, err := client.Do(oreq)
if err != nil {
return err
}
nbody, err := ioutil.ReadAll(nresp.Body)
if err != nil {
return err
}
pretty = strings.Replace(string(nbody), ",\"", ",\n\"", -1)
fmt.Printf("New Settings:\n%s\n", string(pretty))
}
}
return err
}
