func GetStatsForIndexerHttpAddress(indexerHttpAddr, serverUserName, serverPassword string) map[string]interface{} { client := &http.Client{} address := "http://" + indexerHttpAddr + "/stats" req, _ := http.NewRequest("GET", address, nil) req.SetBasicAuth(serverUserName, serverPassword) req.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") resp, err := client.Do(req) if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { log.Printf(address) log.Printf("%v", req) log.Printf("%v", resp) log.Printf("Get stats failed\n") } // todo : error out if response is error tc.HandleError(err, "Get Stats") defer resp.Body.Close() response := make(map[string]interface{}) body, _ := ioutil.ReadAll(resp.Body) err = json.Unmarshal(body, &response) if err != nil { tc.HandleError(err, "Get Bucket :: Unmarshal of response body") } return response }
func GetIndexHttpPort(indexHostAddress, serverUserName, serverPassword, hostaddress string) string { client := &http.Client{} address := "http://" + hostaddress + "/pools/default/nodeServices" req, _ := http.NewRequest("GET", address, nil) req.SetBasicAuth(serverUserName, serverPassword) req.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") resp, err := client.Do(req) if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { log.Printf(address) log.Printf("%v", req) log.Printf("%v", resp) log.Printf("Get indexStatus failed") } // todo : error out if response is error tc.HandleError(err, "Get Stats") defer resp.Body.Close() response := make(map[string]interface{}) body, _ := ioutil.ReadAll(resp.Body) err = json.Unmarshal(body, &response) if err != nil { tc.HandleError(err, "Get nodeServices :: Unmarshal of response body") return "" } log.Printf("%v", response) return "" }
func GetItemCountInBucket(bucketName, bucketPassword, serverUserName, serverPassword, hostaddress string) int { client := &http.Client{} address := "http://" + hostaddress + "/pools/default/buckets/" + bucketName req, _ := http.NewRequest("GET", address, nil) req.SetBasicAuth(serverUserName, serverPassword) req.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") resp, err := client.Do(req) if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { log.Printf(address) log.Printf("%v", req) log.Printf("%v", resp) log.Printf("Get Bucket failed for bucket %v \n", bucketName) } // todo : error out if response is error tc.HandleError(err, "Get Bucket") defer resp.Body.Close() response := make(map[string]interface{}) body, _ := ioutil.ReadAll(resp.Body) err = json.Unmarshal(body, &response) if err != nil { tc.HandleError(err, "Get Bucket :: Unmarshal of response body") } var itemcount float64 basicstats := response["basicStats"].(map[string]interface{}) itemcount = basicstats["itemCount"].(float64) return int(itemcount) }
func DropSecondaryIndex(indexName, bucketName, server string) error { log.Printf("Dropping the secondary index %v", indexName) client, e := CreateClient(server, "2itest") if e != nil { return e } defer client.Close() indexes, err := client.Refresh() tc.HandleError(err, "Error while listing the secondary indexes") for _, index := range indexes { defn := index.Definition if (defn.Name == indexName) && (defn.Bucket == bucketName) { start := time.Now() e := client.DropIndex(uint64(defn.DefnId)) elapsed := time.Since(start) if e == nil { log.Printf("Index dropped") tc.LogPerfStat("DropIndex", elapsed) } else { return e } } } return nil }
func LoadJSONFromCompressedFile(path, docidfield string) tc.KeyValues { file, err := tc.ReadCompressedFile(path) tc.HandleError(err, "Error while decompressing data file "+path) var data interface{} json.Unmarshal(file, &data) m := data.([]interface{}) keyValues := make(tc.KeyValues) var i = 0 var k string if len(docidfield) > 0 { for _, v := range m { k = fmt.Sprintf("%v", v.(map[string]interface{})[docidfield]) keyValues[k] = v i++ } } else { for _, v := range m { k = fmt.Sprintf("%v", i) keyValues[k] = v i++ } } return keyValues }
func Get(key string, rv interface{}, bucketName string, password string, hostaddress string) { url := "http://" + bucketName + ":" + password + "@" + hostaddress c, err := couchbase.Connect(url) tc.HandleError(err, "connect - "+url) p, err := c.GetPool("default") tc.HandleError(err, "pool") b, err := p.GetBucket("test") tc.HandleError(err, "bucket") err = b.Get(key, &rv) tc.HandleError(err, "get") }
func init() { log.Printf("In init()") logging.SetLogLevel(logging.Warn) var configpath string flag.StringVar(&configpath, "cbconfig", "../config/clusterrun_conf.json", "Path of the configuration file with data about Couchbase Cluster") flag.Parse() clusterconfig = tc.GetClusterConfFromFile(configpath) kvaddress = clusterconfig.KVAddress indexManagementAddress = clusterconfig.KVAddress indexScanAddress = clusterconfig.KVAddress seed = 1 proddir, bagdir = tc.FetchMonsterToolPath() // setup cbauth if _, err := cbauth.InternalRetryDefaultInit(kvaddress, clusterconfig.Username, clusterconfig.Password); err != nil { log.Fatalf("Failed to initialize cbauth: %s", err) } secondaryindex.CheckCollation = true e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) tc.HandleError(e, "Error in DropAllSecondaryIndexes") time.Sleep(5 * time.Second) // Working with Users10k and Users_mut dataset. u, _ := user.Current() dataFilePath = filepath.Join(u.HomeDir, "testdata/Users10k.txt.gz") mutationFilePath = filepath.Join(u.HomeDir, "testdata/Users_mut.txt.gz") tc.DownloadDataFile(tc.IndexTypesStaticJSONDataS3, dataFilePath, true) tc.DownloadDataFile(tc.IndexTypesMutationJSONDataS3, mutationFilePath, true) docs = datautility.LoadJSONFromCompressedFile(dataFilePath, "docid") mut_docs = datautility.LoadJSONFromCompressedFile(mutationFilePath, "docid") log.Printf("Emptying the default bucket") kvutility.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress) kvutility.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress) time.Sleep(5 * time.Second) log.Printf("Create Index On the empty default Bucket()") var indexName = "index_eyeColor" var bucketName = "default" err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"eyeColor"}, false, nil, true, defaultIndexActiveTimeout, nil) tc.HandleError(err, "Error in creating the index") // Populate the bucket now log.Printf("Populating the default bucket") kvutility.SetKeyValues(docs, "default", "", clusterconfig.KVAddress) }
func Delete(key string, bucketName string, password string, hostaddress string) { url := "http://" + bucketName + ":" + password + "@" + hostaddress c, err := couchbase.Connect(url) tc.HandleError(err, "connect - "+url) p, err := c.GetPool("default") tc.HandleError(err, "pool") b, err := p.GetBucket(bucketName) tc.HandleError(err, "bucket") err = b.Delete(key) tc.HandleError(err, "delete") b.Close() }
func SetKeyValues(keyValues tc.KeyValues, bucketName string, password string, hostaddress string) { url := "http://" + bucketName + ":" + password + "@" + hostaddress c, err := couchbase.Connect(url) tc.HandleError(err, "connect - "+url) p, err := c.GetPool("default") tc.HandleError(err, "pool") b, err := p.GetBucket(bucketName) tc.HandleError(err, "bucket") for key, value := range keyValues { err = b.Set(key, 0, value) tc.HandleError(err, "set") } b.Close() }
func GetIndexHostNode(indexName, bucketName, serverUserName, serverPassword, hostaddress string) (string, error) { client := &http.Client{} address := "http://" + hostaddress + "/indexStatus" req, _ := http.NewRequest("GET", address, nil) req.SetBasicAuth(serverUserName, serverPassword) req.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") resp, err := client.Do(req) if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { log.Printf(address) log.Printf("%v", req) log.Printf("%v", resp) log.Printf("Get indexStatus failed") } // todo : error out if response is error tc.HandleError(err, "Get Stats") defer resp.Body.Close() response := make(map[string]interface{}) body, _ := ioutil.ReadAll(resp.Body) err = json.Unmarshal(body, &response) if err != nil { tc.HandleError(err, "Get IndexStatus :: Unmarshal of response body") return "", nil } c, e := CreateClient(hostaddress, "2itest") if e != nil { return "", e } defer c.Close() defnID, _ := GetDefnID(c, bucketName, indexName) indexes := response["indexes"].([]interface{}) for _, index := range indexes { i := index.(map[string]interface{}) if i["id"].(float64) == float64(defnID) { hosts := i["hosts"].([]interface{}) return hosts[0].(string), nil } } return "", errors.New("Index not found in /indexStatus") }
func GetDefnID(client *qc.GsiClient, bucket, indexName string) (defnID uint64, ok bool) { indexes, err := client.Refresh() tc.HandleError(err, "Error while listing the indexes") for _, index := range indexes { defn := index.Definition if defn.Bucket == bucket && defn.Name == indexName { return uint64(index.Definition.DefnId), true } } return uint64(c.IndexDefnId(0)), false }
func IndexExistsWithClient(indexName, bucketName, server string, client *qc.GsiClient) bool { indexes, err := client.Refresh() tc.HandleError(err, "Error while listing the secondary indexes") for _, index := range indexes { defn := index.Definition if defn.Name == indexName && defn.Bucket == bucketName { log.Printf("Index found: %v", indexName) return true } } return false }
func Lookup(indexName, bucketName, server string, values []interface{}, distinct bool, limit int64, consistency c.Consistency, vector *qc.TsConsistency) (tc.ScanResponse, error) { var scanErr error scanErr = nil // ToDo: Create a client pool client, e := CreateClient(server, "2itest") if e != nil { return nil, e } defer client.Close() defnID, _ := GetDefnID(client, bucketName, indexName) scanResults := make(tc.ScanResponse) start := time.Now() connErr := client.Lookup( defnID, []c.SecondaryKey{values}, distinct, limit, consistency, vector, func(response qc.ResponseReader) bool { if err := response.Error(); err != nil { scanErr = err return false } else if skeys, pkeys, err := response.GetEntries(); err != nil { scanErr = err return false } else { for i, skey := range skeys { primaryKey := string(pkeys[i]) if _, keyPresent := scanResults[primaryKey]; keyPresent { // Duplicate primary key found tc.HandleError(err, "Duplicate primary key found in the scan results: "+primaryKey) } else { scanResults[primaryKey] = skey } } return true } return false }) elapsed := time.Since(start) if connErr != nil { log.Printf("Connection error in Scan occured: %v", connErr) return scanResults, connErr } else if scanErr != nil { return scanResults, scanErr } tc.LogPerfStat("Lookup", elapsed) return scanResults, nil }
func DeleteBucket(bucketName, bucketPassword, serverUserName, serverPassword, hostaddress string) { client := &http.Client{} address := "http://" + hostaddress + "/pools/default/buckets/" + bucketName req, _ := http.NewRequest("DELETE", address, nil) req.SetBasicAuth(serverUserName, serverPassword) req.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") resp, err := client.Do(req) if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { log.Printf(address) log.Printf("%v", req) log.Printf("%v", resp) log.Printf("DeleteBucket failed for bucket %v \n", bucketName) } // todo : error out if response is error tc.HandleError(err, "Delete Bucket "+address) log.Printf("Deleted bucket %v", bucketName) }
func CreateBucket(bucketName, authenticationType, saslBucketPassword, serverUserName, serverPassword, hostaddress, bucketRamQuota, proxyPort string) { client := &http.Client{} address := "http://" + hostaddress + "/pools/default/buckets" data := url.Values{"name": {bucketName}, "ramQuotaMB": {bucketRamQuota}, "authType": {authenticationType}, "saslPassword": {saslBucketPassword}, "flushEnabled": {"1"}, "replicaNumber": {"1"}, "proxyPort": {proxyPort}} req, _ := http.NewRequest("POST", address, strings.NewReader(data.Encode())) req.SetBasicAuth(serverUserName, serverPassword) req.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") resp, err := client.Do(req) if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { log.Printf(address) log.Printf("%v", req) log.Printf("%v", resp) log.Printf("CreateBucket failed for bucket %v \n", bucketName) } // todo : error out if response is error tc.HandleError(err, "Create Bucket") log.Printf("Created bucket %v", bucketName) }
func IndexExists(indexName, bucketName, server string) (bool, error) { client, e := CreateClient(server, "2itest") if e != nil { return false, e } defer client.Close() indexes, err := client.Refresh() tc.HandleError(err, "Error while listing the secondary indexes") for _, index := range indexes { defn := index.Definition if defn.Name == indexName && defn.Bucket == bucketName { log.Printf("Index found: %v", indexName) return true, nil } } return false, nil }
func RangeWithClient(indexName, bucketName, server string, low, high []interface{}, inclusion uint32, distinct bool, limit int64, consistency c.Consistency, vector *qc.TsConsistency, client *qc.GsiClient) (tc.ScanResponse, error) { var scanErr error scanErr = nil defnID, _ := GetDefnID(client, bucketName, indexName) scanResults := make(tc.ScanResponse) start := time.Now() connErr := client.Range( defnID, c.SecondaryKey(low), c.SecondaryKey(high), qc.Inclusion(inclusion), distinct, limit, consistency, vector, func(response qc.ResponseReader) bool { if err := response.Error(); err != nil { scanErr = err return false } else if skeys, pkeys, err := response.GetEntries(); err != nil { scanErr = err return false } else { for i, skey := range skeys { primaryKey := string(pkeys[i]) if _, keyPresent := scanResults[primaryKey]; keyPresent { // Duplicate primary key found tc.HandleError(err, "Duplicate primary key found in the scan results: "+primaryKey) } else { scanResults[primaryKey] = skey } } return true } return false }) elapsed := time.Since(start) if connErr != nil { log.Printf("Connection error in Scan occured: %v", connErr) return scanResults, connErr } else if scanErr != nil { return scanResults, scanErr } tc.LogPerfStat("Range", elapsed) return scanResults, nil }
func EditBucket(bucketName, bucketPassword, serverUserName, serverPassword, hostaddress, bucketRamQuota string) { client := &http.Client{} address := "http://" + hostaddress + "/pools/default/buckets/" + bucketName data := url.Values{"name": {bucketName}, "ramQuotaMB": {bucketRamQuota}} req, _ := http.NewRequest("POST", address, strings.NewReader(data.Encode())) req.SetBasicAuth(serverUserName, serverPassword) req.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") resp, err := client.Do(req) if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { log.Printf(address) log.Printf("%v", req) log.Printf("%v", resp) log.Printf("EditBucket failed for bucket %v \n", bucketName) } // todo : error out if response is error tc.HandleError(err, "Edit Bucket") time.Sleep(3 * time.Second) log.Printf("Modified parameters of bucket %v", bucketName) }
func DropAllSecondaryIndexes(server string) error { log.Printf("In DropAllSecondaryIndexes()") client, e := CreateClient(server, "2itest") if e != nil { return e } defer client.Close() indexes, err := client.Refresh() tc.HandleError(err, "Error while listing the secondary indexes") for _, index := range indexes { defn := index.Definition // start := time.Now() e := client.DropIndex(uint64(defn.DefnId)) // elapsed := time.Since(start) if e != nil { return e } log.Printf("Dropped index %v", defn.Name) // tc.LogPerfStat("DropIndex", elapsed) // Commenting out this log as it is used only in setup } return nil }
func Range(indexName, bucketName, server string, low, high []interface{}, inclusion uint32, distinct bool, limit int64, consistency c.Consistency, vector *qc.TsConsistency) (tc.ScanResponse, error) { var scanErr error scanErr = nil var previousSecKey value.Value // ToDo: Create a client pool client, e := CreateClient(server, "2itest") if e != nil { return nil, e } defer client.Close() defnID, _ := GetDefnID(client, bucketName, indexName) scanResults := make(tc.ScanResponse) start := time.Now() connErr := client.Range( defnID, c.SecondaryKey(low), c.SecondaryKey(high), qc.Inclusion(inclusion), distinct, limit, consistency, vector, func(response qc.ResponseReader) bool { if err := response.Error(); err != nil { scanErr = err return false } else if skeys, pkeys, err := response.GetEntries(); err != nil { scanErr = err return false } else { for i, skey := range skeys { primaryKey := string(pkeys[i]) if _, keyPresent := scanResults[primaryKey]; keyPresent { // Duplicate primary key found tc.HandleError(err, "Duplicate primary key found in the scan results: "+primaryKey) } else { // Test collation only if CheckCollation is true if CheckCollation == true && len(skey) > 0 { secVal := skey2Values(skey)[0] if previousSecKey == nil { previousSecKey = secVal } else { if secVal.Collate(previousSecKey) < 0 { errMsg := "Collation check failed. Previous Sec key > Current Sec key" scanErr = errors.New(errMsg) return false } } } scanResults[primaryKey] = skey } } return true } return false }) elapsed := time.Since(start) if connErr != nil { log.Printf("Connection error in Scan occured: %v", connErr) return scanResults, connErr } else if scanErr != nil { return scanResults, scanErr } tc.LogPerfStat("Range", elapsed) return scanResults, nil }