func TestMixedDatatypesRange_Float(t *testing.T) { log.Printf("In TestMixedDatatypesRange_Float()") field := "mixed_field" indexName := "index_mixeddt" bucketName := "default" e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) FailTestIfError(e, "Error in DropAllSecondaryIndexes", t) docsToCreate := generateJSONSMixedDatatype(1000, field) seed++ log.Printf("Setting mixed datatypes JSON docs in KV") kvutility.SetKeyValues(docsToCreate, "default", "", clusterconfig.KVAddress) err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{field}, false, nil, true, defaultIndexActiveTimeout, nil) FailTestIfError(err, "Error in creating the index", t) docScanResults := datautility.ExpectedScanResponse_float64(docs, field, 100, 1000, 3) scanResults, err := secondaryindex.Range(indexName, bucketName, indexScanAddress, []interface{}{100}, []interface{}{1000}, 3, true, defaultlimit, c.SessionConsistency, nil) FailTestIfError(err, "Error in scan", t) err = tv.Validate(docScanResults, scanResults) FailTestIfError(err, "Error in scan result validation", t) log.Printf("Lengths of expected and actual scan results are: %d and %d", len(docScanResults), len(scanResults)) docScanResults = datautility.ExpectedScanResponse_float64(docs, field, 1, 100, 2) scanResults, err = secondaryindex.Range(indexName, bucketName, indexScanAddress, []interface{}{1}, []interface{}{100}, 2, true, defaultlimit, c.SessionConsistency, nil) FailTestIfError(err, "Error in scan", t) err = tv.Validate(docScanResults, scanResults) FailTestIfError(err, "Error in scan result validation", t) log.Printf("Lengths of expected and actual scan results are: %d and %d", len(docScanResults), len(scanResults)) log.Printf("Length of kv docs is %d", len(docs)) }
func SkipTestPerfRangeWithoutMutations_1M(t *testing.T) { log.Printf("In TestPerfQueryWithoutMutations_1M()") e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) FailTestIfError(e, "Error in DropAllSecondaryIndexes", t) prodfile := "../../../../../prataprc/monster/prods/test.prod" bagdir := "../../../../../prataprc/monster/bags/" log.Printf("Generating JSON docs") count := 1000000 keyValues := GenerateJsons(count, 1, prodfile, bagdir) log.Printf("Setting JSON docs in KV") kv.SetKeyValues(keyValues, "default", "", clusterconfig.KVAddress) var indexName = "index_company" var bucketName = "default" log.Printf("Creating a 2i") err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"company"}, false, nil, true, 1500, nil) FailTestIfError(err, "Error in creating the index", t) start := time.Now() scanResults, err := secondaryindex.Range(indexName, bucketName, indexScanAddress, []interface{}{"E"}, []interface{}{"N"}, 3, true, defaultlimit, c.AnyConsistency, nil) elapsed := time.Since(start) log.Printf("RangeScan:: Len of scanResults is: %d", len(scanResults)) log.Printf("Range Scan of %d user documents took %s\n", count, elapsed) }
func TestRangeWithConcurrentAddMuts(t *testing.T) { log.Printf("In TestRangeWithConcurrentAddMuts()") var wg sync.WaitGroup prodfile = filepath.Join(proddir, "test.prod") e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) FailTestIfError(e, "Error in DropAllSecondaryIndexes", t) log.Printf("Generating JSON docs") kvdocs = GenerateJsons(1000, seed, prodfile, bagdir) seed++ log.Printf("Setting initial JSON docs in KV") kv.SetKeyValues(kvdocs, "default", "", clusterconfig.KVAddress) var indexName = "index_company" var bucketName = "default" log.Printf("Creating a 2i") err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"company"}, false, nil, true, defaultIndexActiveTimeout, nil) FailTestIfError(err, "Error in creating the index", t) wg.Add(2) go CreateDocsForDuration(&wg, 120) go RangeScanForDuration_ltr("Thread 1: ", &wg, 120, t, indexName, bucketName, indexScanAddress) wg.Wait() }
func TestMixedDatatypesScanAll(t *testing.T) { log.Printf("In TestMixedDatatypesScanAll()") log.Printf("Before test begin: Length of kv docs is %d", len(docs)) field := "md_street" indexName := "index_mixeddt" bucketName := "default" e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) FailTestIfError(e, "Error in DropAllSecondaryIndexes", t) docsToCreate := generateJSONSMixedDatatype(1000, "md_street") log.Printf("After generate docs: Length of kv docs is %d", len(docs)) seed++ log.Printf("Setting mixed datatypes JSON docs in KV") kvutility.SetKeyValues(docsToCreate, "default", "", clusterconfig.KVAddress) err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{field}, false, nil, true, defaultIndexActiveTimeout, nil) FailTestIfError(err, "Error in creating the index", t) docScanResults := datautility.ExpectedScanAllResponse(docs, field) scanResults, err := secondaryindex.ScanAll(indexName, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil) FailTestIfError(err, "Error in scan", t) err = tv.Validate(docScanResults, scanResults) FailTestIfError(err, "Error in scan result validation", t) log.Printf("Lengths of expected and actual scan results are: %d and %d", len(docScanResults), len(scanResults)) log.Printf("End: Length of kv docs is %d", len(docs)) }
// Test case for testing primary key values with longest length possible func TestLargePrimaryKeyLength(t *testing.T) { log.Printf("In TestLargePrimaryKeyLength()") indexName := "index_LongPrimaryField" bucketName := "default" e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) FailTestIfError(e, "Error in DropAllSecondaryIndexes", t) largePrimaryKeyDocs := generateLargePrimaryKeyDocs(1000, "docid") seed++ log.Printf("Setting JSON docs in KV") kvutility.SetKeyValues(largePrimaryKeyDocs, "default", "", clusterconfig.KVAddress) // Create a primary index err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", nil, true, nil, true, defaultIndexActiveTimeout, nil) FailTestIfError(err, "Error in creating the index", t) scanResults, err := secondaryindex.ScanAll(indexName, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil) FailTestIfError(err, "Error in scan", t) if len(scanResults) != len(docs) { log.Printf("Len of scanResults is incorrect. Expected and Actual are: %d and %d", len(docs), len(scanResults)) err = errors.New("Len of scanResults is incorrect.") FailTestIfError(err, "Len of scanResults is incorrect", t) } log.Printf("Lengths of num of docs and scanResults are: %d and %d", len(docs), len(scanResults)) log.Printf("End: Length of kv docs is %d", len(docs)) }
// Test case for testing secondary key field values as very huge func TestLargeSecondaryKeyLength(t *testing.T) { log.Printf("In TestLargeSecondaryKeyLength()") field := "LongSecField" indexName := "index_LongSecField" bucketName := "default" e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) FailTestIfError(e, "Error in DropAllSecondaryIndexes", t) largeKeyDocs := generateLargeSecondayKeyDocs(1000, field) seed++ log.Printf("Setting JSON docs in KV") kvutility.SetKeyValues(largeKeyDocs, "default", "", clusterconfig.KVAddress) err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{field}, false, nil, true, defaultIndexActiveTimeout, nil) FailTestIfError(err, "Error in creating the index", t) docScanResults := datautility.ExpectedScanAllResponse(docs, field) scanResults, err := secondaryindex.ScanAll(indexName, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil) FailTestIfError(err, "Error in scan", t) log.Printf("ScanAll: Lengths of expected and actual scan results are: %d and %d", len(docScanResults), len(scanResults)) err = tv.Validate(docScanResults, scanResults) FailTestIfError(err, "Error in scan result validation", t) docScanResults = datautility.ExpectedScanResponse_string(docs, field, "A", "zzzz", 3) scanResults, err = secondaryindex.Range(indexName, bucketName, indexScanAddress, []interface{}{"A"}, []interface{}{"zzzz"}, 3, true, defaultlimit, c.SessionConsistency, nil) FailTestIfError(err, "Error in scan", t) log.Printf("Range: Lengths of expected and actual scan results are: %d and %d", len(docScanResults), len(scanResults)) err = tv.Validate(docScanResults, scanResults) FailTestIfError(err, "Error in scan result validation", t) log.Printf("End: Length of kv docs is %d", len(docs)) }
func TestMutationsWithMultipleIndexBuilds(t *testing.T) { log.Printf("In TestMutationsWithMultipleIndexBuilds()") prodfile = filepath.Join(proddir, "test.prod") secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) log.Printf("Generating JSON docs") kvdocs = GenerateJsons(100000, seed, prodfile, bagdir) seed++ log.Printf("Setting initial JSON docs in KV") kv.SetKeyValues(kvdocs, "default", "", clusterconfig.KVAddress) var index_p = "index_primary" var bucketName = "default" indexes := []string{"index_company", "index_age", "index_firstname", "index_lastname"} fields := []string{"company", "age", "`first-name`", "`last-name`"} // Create a primary index err := secondaryindex.CreateSecondaryIndex(index_p, bucketName, indexManagementAddress, "", nil, true, nil, true, defaultIndexActiveTimeout, nil) FailTestIfError(err, "Error in creating the index", t) log.Printf("Creating multiple indexes in deferred mode") for i := range indexes { e := secondaryindex.CreateSecondaryIndexAsync(indexes[i], bucketName, indexManagementAddress, "", []string{fields[i]}, false, []byte("{\"defer_build\": true}"), true, nil) FailTestIfError(e, "Error in creating the index", t) } log.Printf("Build Indexes and wait for indexes to become active: %v", indexes) err = secondaryindex.BuildIndexes(indexes, bucketName, indexManagementAddress, 2000) FailTestIfError(err, "Error in deferred index build", t) }
func SkipTestPerfPrimaryIndexBuild_SimpleJson(t *testing.T) { log.Printf("In TestPerfInitialIndexBuild()") e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) FailTestIfError(e, "Error in DropAllSecondaryIndexes", t) prodfile = filepath.Join(proddir, "test.prod") log.Printf("Generating JSON docs") count := 100000000 keyValues := GenerateJsons(count, 1, prodfile, bagdir) log.Printf("Setting JSON docs in KV") kv.SetKeyValues(keyValues, "default", "", clusterconfig.KVAddress) var indexName = "index_primary" var bucketName = "default" log.Printf("Creating a 2i") start := time.Now() err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", nil, true, nil, true, 1500, nil) FailTestIfError(err, "Error in creating the index", t) elapsed := time.Since(start) log.Printf("Index build of %d user documents took %s\n", count, elapsed) scanResults, err := secondaryindex.ScanAll(indexName, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil) FailTestIfError(err, "Error in scan", t) if len(scanResults) != len(keyValues) { log.Printf("Len of scanResults is incorrect. Expected and Actual are: %d and %d", len(keyValues), len(scanResults)) err = errors.New("Len of scanResults is incorrect.") } FailTestIfError(err, "Len of scanResults is incorrect", t) }
func SkipTestPerfRangeWithConcurrentKVMuts(t *testing.T) { log.Printf("In TestPerfRangeWithConcurrentKVMuts()") var wg sync.WaitGroup prodfile = "../../../../../prataprc/monster/prods/test.prod" bagdir = "../../../../../prataprc/monster/bags/" e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) FailTestIfError(e, "Error in DropAllSecondaryIndexes", t) log.Printf("Generating JSON docs") kvdocs = GenerateJsons(1000, seed, prodfile, bagdir) seed++ log.Printf("Setting initial JSON docs in KV") kv.SetKeyValues(kvdocs, "default", "", clusterconfig.KVAddress) var indexName = "index_company" var bucketName = "default" log.Printf("Creating a 2i") err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"company"}, false, nil, true, 1500, nil) FailTestIfError(err, "Error in creating the index", t) wg.Add(2) go CreateDeleteDocsForDuration(&wg, 60) go RangeScanForDuration_ltr("Range Thread: ", &wg, 60, t, indexName, bucketName, indexScanAddress) wg.Wait() }
func TestSameIndexNameInTwoBuckets(t *testing.T) { log.Printf("In TestSameIndexNameInTwoBuckets()") e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) FailTestIfError(e, "Error in DropAllSecondaryIndexes", t) numOfBuckets := 2 indexName := "b_idx" indexFields := [...]string{"age", "address.city"} bucketNames := [...]string{"default", "buck2"} proxyPorts := [...]string{"11212", "11213"} var bucketDocs [2]tc.KeyValues // Update default bucket ram to 256 // Create two more buckets with ram 256 each // Add different docs to all 3 buckets // Create index on each of them // Query the indexes kvutility.FlushBucket(bucketNames[0], "", clusterconfig.Username, clusterconfig.Password, kvaddress) kvutility.EditBucket(bucketNames[0], "", clusterconfig.Username, clusterconfig.Password, kvaddress, "256") for i := 1; i < numOfBuckets; i++ { kvutility.CreateBucket(bucketNames[i], "sasl", "", clusterconfig.Username, clusterconfig.Password, kvaddress, "256", proxyPorts[i]) } time.Sleep(30 * time.Second) log.Printf("Generating docs and Populating all the buckets") for i := 0; i < numOfBuckets; i++ { bucketDocs[i] = generateDocs(1000, "users.prod") kvutility.SetKeyValues(bucketDocs[i], bucketNames[i], "", clusterconfig.KVAddress) err := secondaryindex.CreateSecondaryIndex(indexName, bucketNames[i], indexManagementAddress, "", []string{indexFields[i]}, false, nil, true, defaultIndexActiveTimeout, nil) FailTestIfError(err, "Error in creating the index", t) } time.Sleep(3 * time.Second) // Scan index of first bucket docScanResults := datautility.ExpectedScanResponse_float64(bucketDocs[0], indexFields[0], 30, 50, 1) scanResults, err := secondaryindex.Range(indexName, bucketNames[0], indexScanAddress, []interface{}{30}, []interface{}{50}, 1, true, defaultlimit, c.SessionConsistency, nil) FailTestIfError(err, "Error in scan 1", t) err = tv.Validate(docScanResults, scanResults) FailTestIfError(err, "Error in scan result validation", t) // Scan index of second bucket docScanResults = datautility.ExpectedScanResponse_string(bucketDocs[1], indexFields[1], "F", "Q", 2) scanResults, err = secondaryindex.Range(indexName, bucketNames[1], indexScanAddress, []interface{}{"F"}, []interface{}{"Q"}, 2, true, defaultlimit, c.SessionConsistency, nil) FailTestIfError(err, "Error in scan 2", t) err = tv.Validate(docScanResults, scanResults) FailTestIfError(err, "Error in scan result validation", t) kvutility.EditBucket(bucketNames[0], "", clusterconfig.Username, clusterconfig.Password, kvaddress, "512") kvutility.DeleteBucket(bucketNames[1], "", clusterconfig.Username, clusterconfig.Password, kvaddress) time.Sleep(30 * time.Second) // Sleep after bucket create or delete tc.ClearMap(docs) UpdateKVDocs(bucketDocs[0], docs) }
func SkipTestCompactionDiskMinSize(t *testing.T) { log.Printf("In TestCompactionDiskMinSize()") secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) log.Printf("Emptying the default bucket") kv.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress) kv.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress) tc.ClearMap(docs) time.Sleep(5 * time.Second) log.Printf("Generating JSON docs") docs = GenerateJsons(10000, seed, filepath.Join(proddir, "test.prod"), bagdir) seed++ log.Printf("Setting initial JSON docs in KV") kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress) indexName := "index_compactiontest1" bucketName := "default" indexField := "company" min_sizeValue := float64(320000000) err := secondaryindex.ChangeIndexerSettings("indexer.settings.compaction.min_size", min_sizeValue, clusterconfig.Username, clusterconfig.Password, kvaddress) FailTestIfError(err, "Error in ChangeIndexerSettings", t) log.Printf("Creating a 2i") err = secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{indexField}, false, nil, true, defaultIndexActiveTimeout, nil) FailTestIfError(err, "Error in creating the index", t) stats1 := getCompactionStats(indexName, bucketName) log.Printf("Current Compaction Stats: Fragmentation=%v, Num_Compactions=%v, Disk_Size=%v\n", stats1.frag_percent, stats1.num_compactions, stats1.disk_size) prev_compactionnumber := stats1.num_compactions for i := 0; i < 100; i++ { log.Printf("ITERATION %v", i) updateDocsFieldForFragmentation(indexField) kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress) stats2 := getCompactionStats(indexName, bucketName) log.Printf("Current Compaction Stats: Fragmentation=%v, Num_Compactions=%v, Disk_Size=%0.0f\n", stats2.frag_percent, stats2.num_compactions, stats2.disk_size) if stats2.num_compactions > prev_compactionnumber { time.Sleep(10 * time.Second) stats3 := getCompactionStats(indexName, bucketName) if stats3.disk_size < min_sizeValue { errstr := fmt.Sprintf("Index compaction occurred before disk size %0.0f reached the set min_size %0.0f", stats3.disk_size, min_sizeValue) log.Printf(errstr) err := errors.New(errstr) FailTestIfError(err, "Error in TestCompactionDiskMinSize", t) } } prev_compactionnumber = stats2.num_compactions } err = secondaryindex.ChangeIndexerSettings("indexer.settings.compaction.min_size", float64(1048576), clusterconfig.Username, clusterconfig.Password, kvaddress) FailTestIfError(err, "Error in ChangeIndexerSettings", t) }
func compactionFragmentationTest(fragmentationValue float64, updateFragmentationValue bool, indexName, bucketName, indexField string, updateCount int, t *testing.T) { secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) log.Printf("Emptying the default bucket") kv.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress) kv.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress) tc.ClearMap(docs) time.Sleep(5 * time.Second) log.Printf("Generating JSON docs") docs = GenerateJsons(10000, seed, filepath.Join(proddir, "test.prod"), bagdir) seed++ log.Printf("Setting initial JSON docs in KV") kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress) if updateFragmentationValue { err := secondaryindex.ChangeIndexerSettings("indexer.settings.compaction.min_frag", fragmentationValue, clusterconfig.Username, clusterconfig.Password, kvaddress) FailTestIfError(err, "Error in ChangeIndexerSettings", t) } log.Printf("Creating a 2i") err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{indexField}, false, nil, true, defaultIndexActiveTimeout, nil) FailTestIfError(err, "Error in creating the index", t) stats1 := getCompactionStats(indexName, bucketName) log.Printf("Compaction Stats are: Fragmentation = %v and Num of Compactions = %v\n", stats1.frag_percent, stats1.num_compactions) prev_compactionnumber := stats1.num_compactions for i := 0; i < updateCount; i++ { log.Printf("ITERATION %v", i) updateDocsFieldForFragmentation(indexField) kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress) stats2 := getCompactionStats(indexName, bucketName) log.Printf("Current Compaction Stats are: Fragmentation = %v and Num of Compactions = %v\n", stats2.frag_percent, stats2.num_compactions) if stats2.frag_percent > fragmentationValue { time.Sleep(40 * time.Second) stats3 := getCompactionStats(indexName, bucketName) if stats3.num_compactions <= prev_compactionnumber { errorStr := fmt.Sprintf("Expected compaction to occur at %v but did not occur. Number of compactions = %v", fragmentationValue, stats3.num_compactions) log.Printf(errorStr) FailTestIfError(errors.New(errorStr), "Error in TestDefaultCompactionBehavior", t) } stats1 = getCompactionStats(indexName, bucketName) log.Printf("Compaction occured :: Compaction Stats are: Fragmentation = %v and Num of Compactions = %v\n", stats1.frag_percent, stats1.num_compactions) } prev_compactionnumber = stats1.num_compactions } err = secondaryindex.ChangeIndexerSettings("indexer.settings.compaction.min_frag", float64(30), clusterconfig.Username, clusterconfig.Password, kvaddress) FailTestIfError(err, "Error in ChangeIndexerSettings", t) }
// Test with large number of mutations func TestLargeMutations(t *testing.T) { log.Printf("In TestLargeMutations()") var index1 = "indexmut_1" var index2 = "indexmut_2" var bucketName = "default" var field1 = "company" var field2 = "gender" e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) FailTestIfError(e, "Error in DropAllSecondaryIndexes", t) docsToCreate := generateDocs(20000, "users.prod") UpdateKVDocs(docsToCreate, docs) kv.SetKeyValues(docsToCreate, "default", "", clusterconfig.KVAddress) err := secondaryindex.CreateSecondaryIndex(index1, bucketName, indexManagementAddress, "", []string{field1}, false, nil, true, defaultIndexActiveTimeout, nil) FailTestIfError(err, "Error in creating the index", t) docScanResults := datautility.ExpectedScanAllResponse(docs, field1) scanResults, err := secondaryindex.ScanAll(index1, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil) FailTestIfError(err, "Error in scan", t) err = tv.Validate(docScanResults, scanResults) FailTestIfError(err, "Error in scan result validation", t) log.Printf("Len of expected and actual scan results are : %d and %d", len(docScanResults), len(scanResults)) for i := 0; i <= 10; i++ { log.Printf("ITERATION %v\n", i) docsToCreate = generateDocs(10000, "users.prod") UpdateKVDocs(docsToCreate, docs) kv.SetKeyValues(docsToCreate, "default", "", clusterconfig.KVAddress) err := secondaryindex.CreateSecondaryIndex(index2, bucketName, indexManagementAddress, "", []string{field2}, false, nil, true, defaultIndexActiveTimeout, nil) FailTestIfError(err, "Error in creating the index", t) docScanResults = datautility.ExpectedScanAllResponse(docs, field1) scanResults, err = secondaryindex.ScanAll(index1, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil) FailTestIfError(err, "Error in scan", t) err = tv.Validate(docScanResults, scanResults) FailTestIfError(err, "Error in scan result validation", t) log.Printf("Len of expected and actual scan results are : %d and %d", len(docScanResults), len(scanResults)) docScanResults = datautility.ExpectedScanAllResponse(docs, field2) scanResults, err = secondaryindex.ScanAll(index2, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil) FailTestIfError(err, "Error in scan", t) err = tv.Validate(docScanResults, scanResults) FailTestIfError(err, "Error in scan result validation", t) log.Printf("Len of expected and actual scan results are : %d and %d", len(docScanResults), len(scanResults)) err = secondaryindex.DropSecondaryIndex(index2, bucketName, indexManagementAddress) FailTestIfError(err, "Error in drop index", t) } }
func init() { log.Printf("In init()") logging.SetLogLevel(logging.Warn) var configpath string flag.StringVar(&configpath, "cbconfig", "../config/clusterrun_conf.json", "Path of the configuration file with data about Couchbase Cluster") flag.Parse() clusterconfig = tc.GetClusterConfFromFile(configpath) kvaddress = clusterconfig.KVAddress indexManagementAddress = clusterconfig.KVAddress indexScanAddress = clusterconfig.KVAddress seed = 1 proddir, bagdir = tc.FetchMonsterToolPath() // setup cbauth if _, err := cbauth.InternalRetryDefaultInit(kvaddress, clusterconfig.Username, clusterconfig.Password); err != nil { log.Fatalf("Failed to initialize cbauth: %s", err) } secondaryindex.CheckCollation = true e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) tc.HandleError(e, "Error in DropAllSecondaryIndexes") time.Sleep(5 * time.Second) // Working with Users10k and Users_mut dataset. u, _ := user.Current() dataFilePath = filepath.Join(u.HomeDir, "testdata/Users10k.txt.gz") mutationFilePath = filepath.Join(u.HomeDir, "testdata/Users_mut.txt.gz") tc.DownloadDataFile(tc.IndexTypesStaticJSONDataS3, dataFilePath, true) tc.DownloadDataFile(tc.IndexTypesMutationJSONDataS3, mutationFilePath, true) docs = datautility.LoadJSONFromCompressedFile(dataFilePath, "docid") mut_docs = datautility.LoadJSONFromCompressedFile(mutationFilePath, "docid") log.Printf("Emptying the default bucket") kvutility.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress) kvutility.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress) time.Sleep(5 * time.Second) log.Printf("Create Index On the empty default Bucket()") var indexName = "index_eyeColor" var bucketName = "default" err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"eyeColor"}, false, nil, true, defaultIndexActiveTimeout, nil) tc.HandleError(err, "Error in creating the index") // Populate the bucket now log.Printf("Populating the default bucket") kvutility.SetKeyValues(docs, "default", "", clusterconfig.KVAddress) }
func SkipTestSequentialRangeScans(t *testing.T) { log.Printf("In TestSequentialRangeScans()") prodfile = filepath.Join(proddir, "test.prod") e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) FailTestIfError(e, "Error in DropAllSecondaryIndexes", t) log.Printf("Generating JSON docs") kvdocs = GenerateJsons(1000, seed, prodfile, bagdir) seed++ log.Printf("Setting initial JSON docs in KV") kv.SetKeyValues(kvdocs, "default", "", clusterconfig.KVAddress) var indexName = "index_company" var bucketName = "default" log.Printf("Creating a 2i") err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"company"}, false, nil, true, defaultIndexActiveTimeout, nil) FailTestIfError(err, "Error in creating the index", t) SequentialRangeScanForDuration(indexName, bucketName, 60, t) }
// ===================================================== // Stats Tests // ===================================================== func TestStat_ItemsCount(t *testing.T) { log.Printf("In TestStat_ItemsCount()") // Stat Name: items_count secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) log.Printf("Emptying the default bucket") kv.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress) kv.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress) tc.ClearMap(docs) time.Sleep(5 * time.Second) log.Printf("Generating JSON docs") docs = GenerateJsons(10000, seed, filepath.Join(proddir, "test.prod"), bagdir) seed++ log.Printf("Setting initial JSON docs in KV") kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress) indexName := "index_test1" bucketName := "default" indexField := "company" log.Printf("Creating a 2i") err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{indexField}, false, nil, true, defaultIndexActiveTimeout, nil) FailTestIfError(err, "Error in creating the index", t) time.Sleep(10 * time.Second) prefix := bucketName + ":" + indexName stats := secondaryindex.GetIndexStats(indexName, bucketName, clusterconfig.Username, clusterconfig.Password, kvaddress) itemsCount := stats[prefix+":items_count"].(float64) log.Printf("items_count stat is %v", itemsCount) if itemsCount != float64(len(docs)) { log.Printf("Expected number items count = %v, actual items_count stat returned = %v", len(docs), itemsCount) err = errors.New("items_count is incorrect for index " + prefix) FailTestIfError(err, "Error in TestStat_ItemsCount", t) } }
func TestBucketFlush(t *testing.T) { log.Printf("In TestBucketFlush()") var bucketName = "default" indexNames := [...]string{"index_age", "index_gender", "index_city"} indexFields := [...]string{"age", "gender", "address.city"} e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) FailTestIfError(e, "Error in DropAllSecondaryIndexes", t) kvutility.FlushBucket(bucketName, "", clusterconfig.Username, clusterconfig.Password, kvaddress) kvdocs := generateDocs(1000, "users.prod") kvutility.SetKeyValues(kvdocs, bucketName, "", clusterconfig.KVAddress) for i := 0; i < 3; i++ { err := secondaryindex.CreateSecondaryIndex(indexNames[i], bucketName, indexManagementAddress, "", []string{indexFields[i]}, false, nil, true, defaultIndexActiveTimeout, nil) FailTestIfError(err, "Error in creating the index", t) docScanResults := datautility.ExpectedScanAllResponse(kvdocs, indexFields[i]) scanResults, err := secondaryindex.ScanAll(indexNames[i], bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil) err = tv.Validate(docScanResults, scanResults) FailTestIfError(err, "Error in scan result validation", t) } kvutility.FlushBucket(bucketName, "", clusterconfig.Username, clusterconfig.Password, kvaddress) time.Sleep(5 * time.Second) log.Printf("TestBucketFlush:: Flushed the bucket") for i := 0; i < 3; i++ { scanResults, err := secondaryindex.ScanAll(indexNames[i], bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil) FailTestIfError(err, "Error in scan ", t) if len(scanResults) != 0 { log.Printf("Scan results should be empty after bucket but it is : %v\n", scanResults) e := errors.New("Scan failed after bucket flush") FailTestIfError(e, "Error in TestBucketFlush", t) } } tc.ClearMap(docs) }
func SkipTestPerfInitialIndexBuild_SimpleJson(t *testing.T) { log.Printf("In TestPerfInitialIndexBuild()") e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress) FailTestIfError(e, "Error in DropAllSecondaryIndexes", t) prodfile = filepath.Join(proddir, "test.prod") log.Printf("Generating JSON docs") count := 1000000 keyValues := GenerateJsons(count, 1, prodfile, bagdir) log.Printf("Setting JSON docs in KV") kv.SetKeyValues(keyValues, "default", "", clusterconfig.KVAddress) var indexName = "index_company" var bucketName = "default" log.Printf("Creating a 2i") start := time.Now() err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"company"}, false, nil, true, 1500, nil) FailTestIfError(err, "Error in creating the index", t) elapsed := time.Since(start) log.Printf("Index build of %d user documents took %s\n", count, elapsed) }