func SkipTestCompactionDiskMinSize(t *testing.T) {
log.Printf("In TestCompactionDiskMinSize()")
secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
log.Printf("Emptying the default bucket")
kv.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
kv.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
tc.ClearMap(docs)
time.Sleep(5 * time.Second)
log.Printf("Generating JSON docs")
docs = GenerateJsons(10000, seed, filepath.Join(proddir, "test.prod"), bagdir)
seed++
log.Printf("Setting initial JSON docs in KV")
kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)
indexName := "index_compactiontest1"
bucketName := "default"
indexField := "company"
min_sizeValue := float64(320000000)
err := secondaryindex.ChangeIndexerSettings("indexer.settings.compaction.min_size", min_sizeValue, clusterconfig.Username, clusterconfig.Password, kvaddress)
FailTestIfError(err, "Error in ChangeIndexerSettings", t)
log.Printf("Creating a 2i")
err = secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{indexField}, false, nil, true, defaultIndexActiveTimeout, nil)
FailTestIfError(err, "Error in creating the index", t)
stats1 := getCompactionStats(indexName, bucketName)
log.Printf("Current Compaction Stats: Fragmentation=%v, Num_Compactions=%v, Disk_Size=%v\n", stats1.frag_percent, stats1.num_compactions, stats1.disk_size)
prev_compactionnumber := stats1.num_compactions
for i := 0; i < 100; i++ {
log.Printf("ITERATION %v", i)
updateDocsFieldForFragmentation(indexField)
kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)
stats2 := getCompactionStats(indexName, bucketName)
log.Printf("Current Compaction Stats: Fragmentation=%v, Num_Compactions=%v, Disk_Size=%0.0f\n", stats2.frag_percent, stats2.num_compactions, stats2.disk_size)
if stats2.num_compactions > prev_compactionnumber {
time.Sleep(10 * time.Second)
stats3 := getCompactionStats(indexName, bucketName)
if stats3.disk_size < min_sizeValue {
errstr := fmt.Sprintf("Index compaction occurred before disk size %0.0f reached the set min_size %0.0f", stats3.disk_size, min_sizeValue)
log.Printf(errstr)
err := errors.New(errstr)
FailTestIfError(err, "Error in TestCompactionDiskMinSize", t)
}
}
prev_compactionnumber = stats2.num_compactions
}
err = secondaryindex.ChangeIndexerSettings("indexer.settings.compaction.min_size", float64(1048576), clusterconfig.Username, clusterconfig.Password, kvaddress)
FailTestIfError(err, "Error in ChangeIndexerSettings", t)
}
func compactionFragmentationTest(fragmentationValue float64, updateFragmentationValue bool, indexName, bucketName, indexField string, updateCount int, t *testing.T) {
secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
log.Printf("Emptying the default bucket")
kv.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
kv.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
tc.ClearMap(docs)
time.Sleep(5 * time.Second)
log.Printf("Generating JSON docs")
docs = GenerateJsons(10000, seed, filepath.Join(proddir, "test.prod"), bagdir)
seed++
log.Printf("Setting initial JSON docs in KV")
kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)
if updateFragmentationValue {
err := secondaryindex.ChangeIndexerSettings("indexer.settings.compaction.min_frag", fragmentationValue, clusterconfig.Username, clusterconfig.Password, kvaddress)
FailTestIfError(err, "Error in ChangeIndexerSettings", t)
}
log.Printf("Creating a 2i")
err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{indexField}, false, nil, true, defaultIndexActiveTimeout, nil)
FailTestIfError(err, "Error in creating the index", t)
stats1 := getCompactionStats(indexName, bucketName)
log.Printf("Compaction Stats are: Fragmentation = %v and Num of Compactions = %v\n", stats1.frag_percent, stats1.num_compactions)
prev_compactionnumber := stats1.num_compactions
for i := 0; i < updateCount; i++ {
log.Printf("ITERATION %v", i)
updateDocsFieldForFragmentation(indexField)
kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)
stats2 := getCompactionStats(indexName, bucketName)
log.Printf("Current Compaction Stats are: Fragmentation = %v and Num of Compactions = %v\n", stats2.frag_percent, stats2.num_compactions)
if stats2.frag_percent > fragmentationValue {
time.Sleep(40 * time.Second)
stats3 := getCompactionStats(indexName, bucketName)
if stats3.num_compactions <= prev_compactionnumber {
errorStr := fmt.Sprintf("Expected compaction to occur at %v but did not occur. Number of compactions = %v", fragmentationValue, stats3.num_compactions)
log.Printf(errorStr)
FailTestIfError(errors.New(errorStr), "Error in TestDefaultCompactionBehavior", t)
}
stats1 = getCompactionStats(indexName, bucketName)
log.Printf("Compaction occured :: Compaction Stats are: Fragmentation = %v and Num of Compactions = %v\n", stats1.frag_percent, stats1.num_compactions)
}
prev_compactionnumber = stats1.num_compactions
}
err = secondaryindex.ChangeIndexerSettings("indexer.settings.compaction.min_frag", float64(30), clusterconfig.Username, clusterconfig.Password, kvaddress)
FailTestIfError(err, "Error in ChangeIndexerSettings", t)
}
func init() {
log.Printf("In init()")
logging.SetLogLevel(logging.Warn)
var configpath string
flag.StringVar(&configpath, "cbconfig", "../config/clusterrun_conf.json", "Path of the configuration file with data about Couchbase Cluster")
flag.Parse()
clusterconfig = tc.GetClusterConfFromFile(configpath)
kvaddress = clusterconfig.KVAddress
indexManagementAddress = clusterconfig.KVAddress
indexScanAddress = clusterconfig.KVAddress
seed = 1
proddir, bagdir = tc.FetchMonsterToolPath()
// setup cbauth
if _, err := cbauth.InternalRetryDefaultInit(kvaddress, clusterconfig.Username, clusterconfig.Password); err != nil {
log.Fatalf("Failed to initialize cbauth: %s", err)
}
secondaryindex.CheckCollation = true
e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
tc.HandleError(e, "Error in DropAllSecondaryIndexes")
time.Sleep(5 * time.Second)
// Working with Users10k and Users_mut dataset.
u, _ := user.Current()
dataFilePath = filepath.Join(u.HomeDir, "testdata/Users10k.txt.gz")
mutationFilePath = filepath.Join(u.HomeDir, "testdata/Users_mut.txt.gz")
tc.DownloadDataFile(tc.IndexTypesStaticJSONDataS3, dataFilePath, true)
tc.DownloadDataFile(tc.IndexTypesMutationJSONDataS3, mutationFilePath, true)
docs = datautility.LoadJSONFromCompressedFile(dataFilePath, "docid")
mut_docs = datautility.LoadJSONFromCompressedFile(mutationFilePath, "docid")
log.Printf("Emptying the default bucket")
kvutility.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
kvutility.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
time.Sleep(5 * time.Second)
log.Printf("Create Index On the empty default Bucket()")
var indexName = "index_eyeColor"
var bucketName = "default"
err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"eyeColor"}, false, nil, true, defaultIndexActiveTimeout, nil)
tc.HandleError(err, "Error in creating the index")
// Populate the bucket now
log.Printf("Populating the default bucket")
kvutility.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)
}
// =====================================================
// Stats Tests
// =====================================================
func TestStat_ItemsCount(t *testing.T) {
log.Printf("In TestStat_ItemsCount()")
// Stat Name: items_count
secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
log.Printf("Emptying the default bucket")
kv.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
kv.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
tc.ClearMap(docs)
time.Sleep(5 * time.Second)
log.Printf("Generating JSON docs")
docs = GenerateJsons(10000, seed, filepath.Join(proddir, "test.prod"), bagdir)
seed++
log.Printf("Setting initial JSON docs in KV")
kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)
indexName := "index_test1"
bucketName := "default"
indexField := "company"
log.Printf("Creating a 2i")
err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{indexField}, false, nil, true, defaultIndexActiveTimeout, nil)
FailTestIfError(err, "Error in creating the index", t)
time.Sleep(10 * time.Second)
prefix := bucketName + ":" + indexName
stats := secondaryindex.GetIndexStats(indexName, bucketName, clusterconfig.Username, clusterconfig.Password, kvaddress)
itemsCount := stats[prefix+":items_count"].(float64)
log.Printf("items_count stat is %v", itemsCount)
if itemsCount != float64(len(docs)) {
log.Printf("Expected number items count = %v, actual items_count stat returned = %v", len(docs), itemsCount)
err = errors.New("items_count is incorrect for index " + prefix)
FailTestIfError(err, "Error in TestStat_ItemsCount", t)
}
}