func GetStatsForIndexerHttpAddress(indexerHttpAddr, serverUserName, serverPassword string) map[string]interface{} {
client := &http.Client{}
address := "http://" + indexerHttpAddr + "/stats"
req, _ := http.NewRequest("GET", address, nil)
req.SetBasicAuth(serverUserName, serverPassword)
req.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
resp, err := client.Do(req)
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
log.Printf(address)
log.Printf("%v", req)
log.Printf("%v", resp)
log.Printf("Get stats failed\n")
}
// todo : error out if response is error
tc.HandleError(err, "Get Stats")
defer resp.Body.Close()
response := make(map[string]interface{})
body, _ := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(body, &response)
if err != nil {
tc.HandleError(err, "Get Bucket :: Unmarshal of response body")
}
return response
}
func GetIndexHttpPort(indexHostAddress, serverUserName, serverPassword, hostaddress string) string {
client := &http.Client{}
address := "http://" + hostaddress + "/pools/default/nodeServices"
req, _ := http.NewRequest("GET", address, nil)
req.SetBasicAuth(serverUserName, serverPassword)
req.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
resp, err := client.Do(req)
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
log.Printf(address)
log.Printf("%v", req)
log.Printf("%v", resp)
log.Printf("Get indexStatus failed")
}
// todo : error out if response is error
tc.HandleError(err, "Get Stats")
defer resp.Body.Close()
response := make(map[string]interface{})
body, _ := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(body, &response)
if err != nil {
tc.HandleError(err, "Get nodeServices :: Unmarshal of response body")
return ""
}
log.Printf("%v", response)
return ""
}
func GetItemCountInBucket(bucketName, bucketPassword, serverUserName, serverPassword, hostaddress string) int {
client := &http.Client{}
address := "http://" + hostaddress + "/pools/default/buckets/" + bucketName
req, _ := http.NewRequest("GET", address, nil)
req.SetBasicAuth(serverUserName, serverPassword)
req.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
resp, err := client.Do(req)
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
log.Printf(address)
log.Printf("%v", req)
log.Printf("%v", resp)
log.Printf("Get Bucket failed for bucket %v \n", bucketName)
}
// todo : error out if response is error
tc.HandleError(err, "Get Bucket")
defer resp.Body.Close()
response := make(map[string]interface{})
body, _ := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(body, &response)
if err != nil {
tc.HandleError(err, "Get Bucket :: Unmarshal of response body")
}
var itemcount float64
basicstats := response["basicStats"].(map[string]interface{})
itemcount = basicstats["itemCount"].(float64)
return int(itemcount)
}
func DropSecondaryIndex(indexName, bucketName, server string) error {
log.Printf("Dropping the secondary index %v", indexName)
client, e := CreateClient(server, "2itest")
if e != nil {
return e
}
defer client.Close()
indexes, err := client.Refresh()
tc.HandleError(err, "Error while listing the secondary indexes")
for _, index := range indexes {
defn := index.Definition
if (defn.Name == indexName) && (defn.Bucket == bucketName) {
start := time.Now()
e := client.DropIndex(uint64(defn.DefnId))
elapsed := time.Since(start)
if e == nil {
log.Printf("Index dropped")
tc.LogPerfStat("DropIndex", elapsed)
} else {
return e
}
}
}
return nil
}
func LoadJSONFromCompressedFile(path, docidfield string) tc.KeyValues {
file, err := tc.ReadCompressedFile(path)
tc.HandleError(err, "Error while decompressing data file "+path)
var data interface{}
json.Unmarshal(file, &data)
m := data.([]interface{})
keyValues := make(tc.KeyValues)
var i = 0
var k string
if len(docidfield) > 0 {
for _, v := range m {
k = fmt.Sprintf("%v", v.(map[string]interface{})[docidfield])
keyValues[k] = v
i++
}
} else {
for _, v := range m {
k = fmt.Sprintf("%v", i)
keyValues[k] = v
i++
}
}
return keyValues
}
func Get(key string, rv interface{}, bucketName string, password string, hostaddress string) {
url := "http://" + bucketName + ":" + password + "@" + hostaddress
c, err := couchbase.Connect(url)
tc.HandleError(err, "connect - "+url)
p, err := c.GetPool("default")
tc.HandleError(err, "pool")
b, err := p.GetBucket("test")
tc.HandleError(err, "bucket")
err = b.Get(key, &rv)
tc.HandleError(err, "get")
}
func init() {
log.Printf("In init()")
logging.SetLogLevel(logging.Warn)
var configpath string
flag.StringVar(&configpath, "cbconfig", "../config/clusterrun_conf.json", "Path of the configuration file with data about Couchbase Cluster")
flag.Parse()
clusterconfig = tc.GetClusterConfFromFile(configpath)
kvaddress = clusterconfig.KVAddress
indexManagementAddress = clusterconfig.KVAddress
indexScanAddress = clusterconfig.KVAddress
seed = 1
proddir, bagdir = tc.FetchMonsterToolPath()
// setup cbauth
if _, err := cbauth.InternalRetryDefaultInit(kvaddress, clusterconfig.Username, clusterconfig.Password); err != nil {
log.Fatalf("Failed to initialize cbauth: %s", err)
}
secondaryindex.CheckCollation = true
e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
tc.HandleError(e, "Error in DropAllSecondaryIndexes")
time.Sleep(5 * time.Second)
// Working with Users10k and Users_mut dataset.
u, _ := user.Current()
dataFilePath = filepath.Join(u.HomeDir, "testdata/Users10k.txt.gz")
mutationFilePath = filepath.Join(u.HomeDir, "testdata/Users_mut.txt.gz")
tc.DownloadDataFile(tc.IndexTypesStaticJSONDataS3, dataFilePath, true)
tc.DownloadDataFile(tc.IndexTypesMutationJSONDataS3, mutationFilePath, true)
docs = datautility.LoadJSONFromCompressedFile(dataFilePath, "docid")
mut_docs = datautility.LoadJSONFromCompressedFile(mutationFilePath, "docid")
log.Printf("Emptying the default bucket")
kvutility.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
kvutility.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
time.Sleep(5 * time.Second)
log.Printf("Create Index On the empty default Bucket()")
var indexName = "index_eyeColor"
var bucketName = "default"
err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"eyeColor"}, false, nil, true, defaultIndexActiveTimeout, nil)
tc.HandleError(err, "Error in creating the index")
// Populate the bucket now
log.Printf("Populating the default bucket")
kvutility.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)
}
func Delete(key string, bucketName string, password string, hostaddress string) {
url := "http://" + bucketName + ":" + password + "@" + hostaddress
c, err := couchbase.Connect(url)
tc.HandleError(err, "connect - "+url)
p, err := c.GetPool("default")
tc.HandleError(err, "pool")
b, err := p.GetBucket(bucketName)
tc.HandleError(err, "bucket")
err = b.Delete(key)
tc.HandleError(err, "delete")
b.Close()
}
func SetKeyValues(keyValues tc.KeyValues, bucketName string, password string, hostaddress string) {
url := "http://" + bucketName + ":" + password + "@" + hostaddress
c, err := couchbase.Connect(url)
tc.HandleError(err, "connect - "+url)
p, err := c.GetPool("default")
tc.HandleError(err, "pool")
b, err := p.GetBucket(bucketName)
tc.HandleError(err, "bucket")
for key, value := range keyValues {
err = b.Set(key, 0, value)
tc.HandleError(err, "set")
}
b.Close()
}
func GetIndexHostNode(indexName, bucketName, serverUserName, serverPassword, hostaddress string) (string, error) {
client := &http.Client{}
address := "http://" + hostaddress + "/indexStatus"
req, _ := http.NewRequest("GET", address, nil)
req.SetBasicAuth(serverUserName, serverPassword)
req.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
resp, err := client.Do(req)
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
log.Printf(address)
log.Printf("%v", req)
log.Printf("%v", resp)
log.Printf("Get indexStatus failed")
}
// todo : error out if response is error
tc.HandleError(err, "Get Stats")
defer resp.Body.Close()
response := make(map[string]interface{})
body, _ := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(body, &response)
if err != nil {
tc.HandleError(err, "Get IndexStatus :: Unmarshal of response body")
return "", nil
}
c, e := CreateClient(hostaddress, "2itest")
if e != nil {
return "", e
}
defer c.Close()
defnID, _ := GetDefnID(c, bucketName, indexName)
indexes := response["indexes"].([]interface{})
for _, index := range indexes {
i := index.(map[string]interface{})
if i["id"].(float64) == float64(defnID) {
hosts := i["hosts"].([]interface{})
return hosts[0].(string), nil
}
}
return "", errors.New("Index not found in /indexStatus")
}
func GetDefnID(client *qc.GsiClient, bucket, indexName string) (defnID uint64, ok bool) {
indexes, err := client.Refresh()
tc.HandleError(err, "Error while listing the indexes")
for _, index := range indexes {
defn := index.Definition
if defn.Bucket == bucket && defn.Name == indexName {
return uint64(index.Definition.DefnId), true
}
}
return uint64(c.IndexDefnId(0)), false
}
func IndexExistsWithClient(indexName, bucketName, server string, client *qc.GsiClient) bool {
indexes, err := client.Refresh()
tc.HandleError(err, "Error while listing the secondary indexes")
for _, index := range indexes {
defn := index.Definition
if defn.Name == indexName && defn.Bucket == bucketName {
log.Printf("Index found: %v", indexName)
return true
}
}
return false
}
func Lookup(indexName, bucketName, server string, values []interface{}, distinct bool, limit int64, consistency c.Consistency, vector *qc.TsConsistency) (tc.ScanResponse, error) {
var scanErr error
scanErr = nil
// ToDo: Create a client pool
client, e := CreateClient(server, "2itest")
if e != nil {
return nil, e
}
defer client.Close()
defnID, _ := GetDefnID(client, bucketName, indexName)
scanResults := make(tc.ScanResponse)
start := time.Now()
connErr := client.Lookup(
defnID, []c.SecondaryKey{values}, distinct, limit,
consistency, vector,
func(response qc.ResponseReader) bool {
if err := response.Error(); err != nil {
scanErr = err
return false
} else if skeys, pkeys, err := response.GetEntries(); err != nil {
scanErr = err
return false
} else {
for i, skey := range skeys {
primaryKey := string(pkeys[i])
if _, keyPresent := scanResults[primaryKey]; keyPresent {
// Duplicate primary key found
tc.HandleError(err, "Duplicate primary key found in the scan results: "+primaryKey)
} else {
scanResults[primaryKey] = skey
}
}
return true
}
return false
})
elapsed := time.Since(start)
if connErr != nil {
log.Printf("Connection error in Scan occured: %v", connErr)
return scanResults, connErr
} else if scanErr != nil {
return scanResults, scanErr
}
tc.LogPerfStat("Lookup", elapsed)
return scanResults, nil
}
func DeleteBucket(bucketName, bucketPassword, serverUserName, serverPassword, hostaddress string) {
client := &http.Client{}
address := "http://" + hostaddress + "/pools/default/buckets/" + bucketName
req, _ := http.NewRequest("DELETE", address, nil)
req.SetBasicAuth(serverUserName, serverPassword)
req.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
resp, err := client.Do(req)
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
log.Printf(address)
log.Printf("%v", req)
log.Printf("%v", resp)
log.Printf("DeleteBucket failed for bucket %v \n", bucketName)
}
// todo : error out if response is error
tc.HandleError(err, "Delete Bucket "+address)
log.Printf("Deleted bucket %v", bucketName)
}
func CreateBucket(bucketName, authenticationType, saslBucketPassword, serverUserName, serverPassword, hostaddress, bucketRamQuota, proxyPort string) {
client := &http.Client{}
address := "http://" + hostaddress + "/pools/default/buckets"
data := url.Values{"name": {bucketName}, "ramQuotaMB": {bucketRamQuota}, "authType": {authenticationType}, "saslPassword": {saslBucketPassword}, "flushEnabled": {"1"}, "replicaNumber": {"1"}, "proxyPort": {proxyPort}}
req, _ := http.NewRequest("POST", address, strings.NewReader(data.Encode()))
req.SetBasicAuth(serverUserName, serverPassword)
req.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
resp, err := client.Do(req)
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
log.Printf(address)
log.Printf("%v", req)
log.Printf("%v", resp)
log.Printf("CreateBucket failed for bucket %v \n", bucketName)
}
// todo : error out if response is error
tc.HandleError(err, "Create Bucket")
log.Printf("Created bucket %v", bucketName)
}
func IndexExists(indexName, bucketName, server string) (bool, error) {
client, e := CreateClient(server, "2itest")
if e != nil {
return false, e
}
defer client.Close()
indexes, err := client.Refresh()
tc.HandleError(err, "Error while listing the secondary indexes")
for _, index := range indexes {
defn := index.Definition
if defn.Name == indexName && defn.Bucket == bucketName {
log.Printf("Index found: %v", indexName)
return true, nil
}
}
return false, nil
}
func RangeWithClient(indexName, bucketName, server string, low, high []interface{}, inclusion uint32,
distinct bool, limit int64, consistency c.Consistency, vector *qc.TsConsistency, client *qc.GsiClient) (tc.ScanResponse, error) {
var scanErr error
scanErr = nil
defnID, _ := GetDefnID(client, bucketName, indexName)
scanResults := make(tc.ScanResponse)
start := time.Now()
connErr := client.Range(
defnID, c.SecondaryKey(low), c.SecondaryKey(high), qc.Inclusion(inclusion), distinct, limit,
consistency, vector,
func(response qc.ResponseReader) bool {
if err := response.Error(); err != nil {
scanErr = err
return false
} else if skeys, pkeys, err := response.GetEntries(); err != nil {
scanErr = err
return false
} else {
for i, skey := range skeys {
primaryKey := string(pkeys[i])
if _, keyPresent := scanResults[primaryKey]; keyPresent {
// Duplicate primary key found
tc.HandleError(err, "Duplicate primary key found in the scan results: "+primaryKey)
} else {
scanResults[primaryKey] = skey
}
}
return true
}
return false
})
elapsed := time.Since(start)
if connErr != nil {
log.Printf("Connection error in Scan occured: %v", connErr)
return scanResults, connErr
} else if scanErr != nil {
return scanResults, scanErr
}
tc.LogPerfStat("Range", elapsed)
return scanResults, nil
}
func EditBucket(bucketName, bucketPassword, serverUserName, serverPassword, hostaddress, bucketRamQuota string) {
client := &http.Client{}
address := "http://" + hostaddress + "/pools/default/buckets/" + bucketName
data := url.Values{"name": {bucketName}, "ramQuotaMB": {bucketRamQuota}}
req, _ := http.NewRequest("POST", address, strings.NewReader(data.Encode()))
req.SetBasicAuth(serverUserName, serverPassword)
req.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
resp, err := client.Do(req)
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
log.Printf(address)
log.Printf("%v", req)
log.Printf("%v", resp)
log.Printf("EditBucket failed for bucket %v \n", bucketName)
}
// todo : error out if response is error
tc.HandleError(err, "Edit Bucket")
time.Sleep(3 * time.Second)
log.Printf("Modified parameters of bucket %v", bucketName)
}
func DropAllSecondaryIndexes(server string) error {
log.Printf("In DropAllSecondaryIndexes()")
client, e := CreateClient(server, "2itest")
if e != nil {
return e
}
defer client.Close()
indexes, err := client.Refresh()
tc.HandleError(err, "Error while listing the secondary indexes")
for _, index := range indexes {
defn := index.Definition
// start := time.Now()
e := client.DropIndex(uint64(defn.DefnId))
// elapsed := time.Since(start)
if e != nil {
return e
}
log.Printf("Dropped index %v", defn.Name)
// tc.LogPerfStat("DropIndex", elapsed) // Commenting out this log as it is used only in setup
}
return nil
}
func Range(indexName, bucketName, server string, low, high []interface{}, inclusion uint32,
distinct bool, limit int64, consistency c.Consistency, vector *qc.TsConsistency) (tc.ScanResponse, error) {
var scanErr error
scanErr = nil
var previousSecKey value.Value
// ToDo: Create a client pool
client, e := CreateClient(server, "2itest")
if e != nil {
return nil, e
}
defer client.Close()
defnID, _ := GetDefnID(client, bucketName, indexName)
scanResults := make(tc.ScanResponse)
start := time.Now()
connErr := client.Range(
defnID, c.SecondaryKey(low), c.SecondaryKey(high), qc.Inclusion(inclusion), distinct, limit,
consistency, vector,
func(response qc.ResponseReader) bool {
if err := response.Error(); err != nil {
scanErr = err
return false
} else if skeys, pkeys, err := response.GetEntries(); err != nil {
scanErr = err
return false
} else {
for i, skey := range skeys {
primaryKey := string(pkeys[i])
if _, keyPresent := scanResults[primaryKey]; keyPresent {
// Duplicate primary key found
tc.HandleError(err, "Duplicate primary key found in the scan results: "+primaryKey)
} else {
// Test collation only if CheckCollation is true
if CheckCollation == true && len(skey) > 0 {
secVal := skey2Values(skey)[0]
if previousSecKey == nil {
previousSecKey = secVal
} else {
if secVal.Collate(previousSecKey) < 0 {
errMsg := "Collation check failed. Previous Sec key > Current Sec key"
scanErr = errors.New(errMsg)
return false
}
}
}
scanResults[primaryKey] = skey
}
}
return true
}
return false
})
elapsed := time.Since(start)
if connErr != nil {
log.Printf("Connection error in Scan occured: %v", connErr)
return scanResults, connErr
} else if scanErr != nil {
return scanResults, scanErr
}
tc.LogPerfStat("Range", elapsed)
return scanResults, nil
}