func TestSameIndexNameInTwoBuckets(t *testing.T) {
	log.Printf("In TestSameIndexNameInTwoBuckets()")

	e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	FailTestIfError(e, "Error in DropAllSecondaryIndexes", t)

	numOfBuckets := 2
	indexName := "b_idx"
	indexFields := [...]string{"age", "address.city"}
	bucketNames := [...]string{"default", "buck2"}
	proxyPorts := [...]string{"11212", "11213"}
	var bucketDocs [2]tc.KeyValues

	// Update default bucket ram to 256
	// Create two more buckets with ram 256 each
	// Add different docs to all 3 buckets
	// Create index on each of them
	// Query the indexes

	kvutility.FlushBucket(bucketNames[0], "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	kvutility.EditBucket(bucketNames[0], "", clusterconfig.Username, clusterconfig.Password, kvaddress, "256")

	for i := 1; i < numOfBuckets; i++ {
		kvutility.CreateBucket(bucketNames[i], "sasl", "", clusterconfig.Username, clusterconfig.Password, kvaddress, "256", proxyPorts[i])
	}
	time.Sleep(30 * time.Second)

	log.Printf("Generating docs and Populating all the buckets")
	for i := 0; i < numOfBuckets; i++ {
		bucketDocs[i] = generateDocs(1000, "users.prod")
		kvutility.SetKeyValues(bucketDocs[i], bucketNames[i], "", clusterconfig.KVAddress)
		err := secondaryindex.CreateSecondaryIndex(indexName, bucketNames[i], indexManagementAddress, "", []string{indexFields[i]}, false, nil, true, defaultIndexActiveTimeout, nil)
		FailTestIfError(err, "Error in creating the index", t)
	}
	time.Sleep(3 * time.Second)

	// Scan index of first bucket
	docScanResults := datautility.ExpectedScanResponse_float64(bucketDocs[0], indexFields[0], 30, 50, 1)
	scanResults, err := secondaryindex.Range(indexName, bucketNames[0], indexScanAddress, []interface{}{30}, []interface{}{50}, 1, true, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan 1", t)
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)

	// Scan index of second bucket
	docScanResults = datautility.ExpectedScanResponse_string(bucketDocs[1], indexFields[1], "F", "Q", 2)
	scanResults, err = secondaryindex.Range(indexName, bucketNames[1], indexScanAddress, []interface{}{"F"}, []interface{}{"Q"}, 2, true, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan 2", t)
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)

	kvutility.EditBucket(bucketNames[0], "", clusterconfig.Username, clusterconfig.Password, kvaddress, "512")
	kvutility.DeleteBucket(bucketNames[1], "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	time.Sleep(30 * time.Second) // Sleep after bucket create or delete

	tc.ClearMap(docs)
	UpdateKVDocs(bucketDocs[0], docs)
}
func SkipTestCompactionDiskMinSize(t *testing.T) {
	log.Printf("In TestCompactionDiskMinSize()")

	secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	log.Printf("Emptying the default bucket")
	kv.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	kv.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	tc.ClearMap(docs)
	time.Sleep(5 * time.Second)

	log.Printf("Generating JSON docs")
	docs = GenerateJsons(10000, seed, filepath.Join(proddir, "test.prod"), bagdir)
	seed++

	log.Printf("Setting initial JSON docs in KV")
	kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)

	indexName := "index_compactiontest1"
	bucketName := "default"
	indexField := "company"
	min_sizeValue := float64(320000000)

	err := secondaryindex.ChangeIndexerSettings("indexer.settings.compaction.min_size", min_sizeValue, clusterconfig.Username, clusterconfig.Password, kvaddress)
	FailTestIfError(err, "Error in ChangeIndexerSettings", t)

	log.Printf("Creating a 2i")
	err = secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{indexField}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	stats1 := getCompactionStats(indexName, bucketName)
	log.Printf("Current Compaction Stats: Fragmentation=%v, Num_Compactions=%v, Disk_Size=%v\n", stats1.frag_percent, stats1.num_compactions, stats1.disk_size)
	prev_compactionnumber := stats1.num_compactions

	for i := 0; i < 100; i++ {
		log.Printf("ITERATION %v", i)
		updateDocsFieldForFragmentation(indexField)
		kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)

		stats2 := getCompactionStats(indexName, bucketName)
		log.Printf("Current Compaction Stats: Fragmentation=%v, Num_Compactions=%v, Disk_Size=%0.0f\n", stats2.frag_percent, stats2.num_compactions, stats2.disk_size)
		if stats2.num_compactions > prev_compactionnumber {
			time.Sleep(10 * time.Second)
			stats3 := getCompactionStats(indexName, bucketName)
			if stats3.disk_size < min_sizeValue {
				errstr := fmt.Sprintf("Index compaction occurred before disk size %0.0f reached the set min_size %0.0f", stats3.disk_size, min_sizeValue)
				log.Printf(errstr)
				err := errors.New(errstr)
				FailTestIfError(err, "Error in TestCompactionDiskMinSize", t)
			}
		}
		prev_compactionnumber = stats2.num_compactions
	}

	err = secondaryindex.ChangeIndexerSettings("indexer.settings.compaction.min_size", float64(1048576), clusterconfig.Username, clusterconfig.Password, kvaddress)
	FailTestIfError(err, "Error in ChangeIndexerSettings", t)
}
func compactionFragmentationTest(fragmentationValue float64, updateFragmentationValue bool, indexName, bucketName, indexField string, updateCount int, t *testing.T) {
	secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)

	log.Printf("Emptying the default bucket")
	kv.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	kv.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	tc.ClearMap(docs)
	time.Sleep(5 * time.Second)

	log.Printf("Generating JSON docs")
	docs = GenerateJsons(10000, seed, filepath.Join(proddir, "test.prod"), bagdir)
	seed++

	log.Printf("Setting initial JSON docs in KV")
	kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)

	if updateFragmentationValue {
		err := secondaryindex.ChangeIndexerSettings("indexer.settings.compaction.min_frag", fragmentationValue, clusterconfig.Username, clusterconfig.Password, kvaddress)
		FailTestIfError(err, "Error in ChangeIndexerSettings", t)
	}

	log.Printf("Creating a 2i")
	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{indexField}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	stats1 := getCompactionStats(indexName, bucketName)
	log.Printf("Compaction Stats are: Fragmentation =  %v and Num of Compactions = %v\n", stats1.frag_percent, stats1.num_compactions)
	prev_compactionnumber := stats1.num_compactions

	for i := 0; i < updateCount; i++ {
		log.Printf("ITERATION %v", i)
		updateDocsFieldForFragmentation(indexField)
		kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)

		stats2 := getCompactionStats(indexName, bucketName)
		log.Printf("Current Compaction Stats are: Fragmentation =  %v and Num of Compactions = %v\n", stats2.frag_percent, stats2.num_compactions)

		if stats2.frag_percent > fragmentationValue {
			time.Sleep(40 * time.Second)
			stats3 := getCompactionStats(indexName, bucketName)
			if stats3.num_compactions <= prev_compactionnumber {
				errorStr := fmt.Sprintf("Expected compaction to occur at %v but did not occur. Number of compactions = %v", fragmentationValue, stats3.num_compactions)
				log.Printf(errorStr)
				FailTestIfError(errors.New(errorStr), "Error in TestDefaultCompactionBehavior", t)
			}
			stats1 = getCompactionStats(indexName, bucketName)
			log.Printf("Compaction occured :: Compaction Stats are: Fragmentation =  %v and Num of Compactions = %v\n", stats1.frag_percent, stats1.num_compactions)
		}

		prev_compactionnumber = stats1.num_compactions
	}
	err = secondaryindex.ChangeIndexerSettings("indexer.settings.compaction.min_frag", float64(30), clusterconfig.Username, clusterconfig.Password, kvaddress)
	FailTestIfError(err, "Error in ChangeIndexerSettings", t)
}
Beispiel #4
0
func TestBucketFlush(t *testing.T) {
	log.Printf("In TestBucketFlush()")

	var bucketName = "default"
	indexNames := [...]string{"index_age", "index_gender", "index_city"}
	indexFields := [...]string{"age", "gender", "address.city"}

	e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	FailTestIfError(e, "Error in DropAllSecondaryIndexes", t)
	kvutility.FlushBucket(bucketName, "", clusterconfig.Username, clusterconfig.Password, kvaddress)

	kvdocs := generateDocs(1000, "users.prod")
	kvutility.SetKeyValues(kvdocs, bucketName, "", clusterconfig.KVAddress)

	for i := 0; i < 3; i++ {
		err := secondaryindex.CreateSecondaryIndex(indexNames[i], bucketName, indexManagementAddress, "", []string{indexFields[i]}, false, nil, true, defaultIndexActiveTimeout, nil)
		FailTestIfError(err, "Error in creating the index", t)
		docScanResults := datautility.ExpectedScanAllResponse(kvdocs, indexFields[i])
		scanResults, err := secondaryindex.ScanAll(indexNames[i], bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil)
		err = tv.Validate(docScanResults, scanResults)
		FailTestIfError(err, "Error in scan result validation", t)
	}

	kvutility.FlushBucket(bucketName, "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	time.Sleep(5 * time.Second)
	log.Printf("TestBucketFlush:: Flushed the bucket")

	for i := 0; i < 3; i++ {
		scanResults, err := secondaryindex.ScanAll(indexNames[i], bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil)
		FailTestIfError(err, "Error in scan ", t)
		if len(scanResults) != 0 {
			log.Printf("Scan results should be empty after bucket but it is : %v\n", scanResults)
			e := errors.New("Scan failed after bucket flush")
			FailTestIfError(e, "Error in TestBucketFlush", t)
		}
	}

	tc.ClearMap(docs)
}
Beispiel #5
0
func init() {
	log.Printf("In init()")
	logging.SetLogLevel(logging.Warn)

	var configpath string
	flag.StringVar(&configpath, "cbconfig", "../config/clusterrun_conf.json", "Path of the configuration file with data about Couchbase Cluster")
	flag.Parse()
	clusterconfig = tc.GetClusterConfFromFile(configpath)
	kvaddress = clusterconfig.KVAddress
	indexManagementAddress = clusterconfig.KVAddress
	indexScanAddress = clusterconfig.KVAddress
	seed = 1
	proddir, bagdir = tc.FetchMonsterToolPath()

	// setup cbauth
	if _, err := cbauth.InternalRetryDefaultInit(kvaddress, clusterconfig.Username, clusterconfig.Password); err != nil {
		log.Fatalf("Failed to initialize cbauth: %s", err)
	}
	secondaryindex.CheckCollation = true
	e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	tc.HandleError(e, "Error in DropAllSecondaryIndexes")

	time.Sleep(5 * time.Second)
	// Working with Users10k and Users_mut dataset.
	u, _ := user.Current()
	dataFilePath = filepath.Join(u.HomeDir, "testdata/Users10k.txt.gz")
	mutationFilePath = filepath.Join(u.HomeDir, "testdata/Users_mut.txt.gz")
	tc.DownloadDataFile(tc.IndexTypesStaticJSONDataS3, dataFilePath, true)
	tc.DownloadDataFile(tc.IndexTypesMutationJSONDataS3, mutationFilePath, true)
	docs = datautility.LoadJSONFromCompressedFile(dataFilePath, "docid")
	mut_docs = datautility.LoadJSONFromCompressedFile(mutationFilePath, "docid")
	log.Printf("Emptying the default bucket")
	kvutility.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	kvutility.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	time.Sleep(5 * time.Second)

	log.Printf("Create Index On the empty default Bucket()")
	var indexName = "index_eyeColor"
	var bucketName = "default"

	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"eyeColor"}, false, nil, true, defaultIndexActiveTimeout, nil)
	tc.HandleError(err, "Error in creating the index")

	// Populate the bucket now
	log.Printf("Populating the default bucket")
	kvutility.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)
}
// =====================================================
// Stats Tests
// =====================================================
func TestStat_ItemsCount(t *testing.T) {
	log.Printf("In TestStat_ItemsCount()")

	// Stat Name: items_count
	secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	log.Printf("Emptying the default bucket")
	kv.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	kv.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	tc.ClearMap(docs)
	time.Sleep(5 * time.Second)

	log.Printf("Generating JSON docs")
	docs = GenerateJsons(10000, seed, filepath.Join(proddir, "test.prod"), bagdir)
	seed++

	log.Printf("Setting initial JSON docs in KV")
	kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)

	indexName := "index_test1"
	bucketName := "default"
	indexField := "company"

	log.Printf("Creating a 2i")
	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{indexField}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	time.Sleep(10 * time.Second)

	prefix := bucketName + ":" + indexName
	stats := secondaryindex.GetIndexStats(indexName, bucketName, clusterconfig.Username, clusterconfig.Password, kvaddress)
	itemsCount := stats[prefix+":items_count"].(float64)
	log.Printf("items_count stat is %v", itemsCount)

	if itemsCount != float64(len(docs)) {
		log.Printf("Expected number items count = %v, actual items_count stat returned = %v", len(docs), itemsCount)
		err = errors.New("items_count is incorrect for index " + prefix)
		FailTestIfError(err, "Error in TestStat_ItemsCount", t)
	}
}