コード例 #1
0
func SkipTestCompactionDiskMinSize(t *testing.T) {
	log.Printf("In TestCompactionDiskMinSize()")

	secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	log.Printf("Emptying the default bucket")
	kv.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	kv.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	tc.ClearMap(docs)
	time.Sleep(5 * time.Second)

	log.Printf("Generating JSON docs")
	docs = GenerateJsons(10000, seed, filepath.Join(proddir, "test.prod"), bagdir)
	seed++

	log.Printf("Setting initial JSON docs in KV")
	kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)

	indexName := "index_compactiontest1"
	bucketName := "default"
	indexField := "company"
	min_sizeValue := float64(320000000)

	err := secondaryindex.ChangeIndexerSettings("indexer.settings.compaction.min_size", min_sizeValue, clusterconfig.Username, clusterconfig.Password, kvaddress)
	FailTestIfError(err, "Error in ChangeIndexerSettings", t)

	log.Printf("Creating a 2i")
	err = secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{indexField}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	stats1 := getCompactionStats(indexName, bucketName)
	log.Printf("Current Compaction Stats: Fragmentation=%v, Num_Compactions=%v, Disk_Size=%v\n", stats1.frag_percent, stats1.num_compactions, stats1.disk_size)
	prev_compactionnumber := stats1.num_compactions

	for i := 0; i < 100; i++ {
		log.Printf("ITERATION %v", i)
		updateDocsFieldForFragmentation(indexField)
		kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)

		stats2 := getCompactionStats(indexName, bucketName)
		log.Printf("Current Compaction Stats: Fragmentation=%v, Num_Compactions=%v, Disk_Size=%0.0f\n", stats2.frag_percent, stats2.num_compactions, stats2.disk_size)
		if stats2.num_compactions > prev_compactionnumber {
			time.Sleep(10 * time.Second)
			stats3 := getCompactionStats(indexName, bucketName)
			if stats3.disk_size < min_sizeValue {
				errstr := fmt.Sprintf("Index compaction occurred before disk size %0.0f reached the set min_size %0.0f", stats3.disk_size, min_sizeValue)
				log.Printf(errstr)
				err := errors.New(errstr)
				FailTestIfError(err, "Error in TestCompactionDiskMinSize", t)
			}
		}
		prev_compactionnumber = stats2.num_compactions
	}

	err = secondaryindex.ChangeIndexerSettings("indexer.settings.compaction.min_size", float64(1048576), clusterconfig.Username, clusterconfig.Password, kvaddress)
	FailTestIfError(err, "Error in ChangeIndexerSettings", t)
}
コード例 #2
0
func compactionFragmentationTest(fragmentationValue float64, updateFragmentationValue bool, indexName, bucketName, indexField string, updateCount int, t *testing.T) {
	secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)

	log.Printf("Emptying the default bucket")
	kv.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	kv.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	tc.ClearMap(docs)
	time.Sleep(5 * time.Second)

	log.Printf("Generating JSON docs")
	docs = GenerateJsons(10000, seed, filepath.Join(proddir, "test.prod"), bagdir)
	seed++

	log.Printf("Setting initial JSON docs in KV")
	kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)

	if updateFragmentationValue {
		err := secondaryindex.ChangeIndexerSettings("indexer.settings.compaction.min_frag", fragmentationValue, clusterconfig.Username, clusterconfig.Password, kvaddress)
		FailTestIfError(err, "Error in ChangeIndexerSettings", t)
	}

	log.Printf("Creating a 2i")
	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{indexField}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	stats1 := getCompactionStats(indexName, bucketName)
	log.Printf("Compaction Stats are: Fragmentation =  %v and Num of Compactions = %v\n", stats1.frag_percent, stats1.num_compactions)
	prev_compactionnumber := stats1.num_compactions

	for i := 0; i < updateCount; i++ {
		log.Printf("ITERATION %v", i)
		updateDocsFieldForFragmentation(indexField)
		kv.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)

		stats2 := getCompactionStats(indexName, bucketName)
		log.Printf("Current Compaction Stats are: Fragmentation =  %v and Num of Compactions = %v\n", stats2.frag_percent, stats2.num_compactions)

		if stats2.frag_percent > fragmentationValue {
			time.Sleep(40 * time.Second)
			stats3 := getCompactionStats(indexName, bucketName)
			if stats3.num_compactions <= prev_compactionnumber {
				errorStr := fmt.Sprintf("Expected compaction to occur at %v but did not occur. Number of compactions = %v", fragmentationValue, stats3.num_compactions)
				log.Printf(errorStr)
				FailTestIfError(errors.New(errorStr), "Error in TestDefaultCompactionBehavior", t)
			}
			stats1 = getCompactionStats(indexName, bucketName)
			log.Printf("Compaction occured :: Compaction Stats are: Fragmentation =  %v and Num of Compactions = %v\n", stats1.frag_percent, stats1.num_compactions)
		}

		prev_compactionnumber = stats1.num_compactions
	}
	err = secondaryindex.ChangeIndexerSettings("indexer.settings.compaction.min_frag", float64(30), clusterconfig.Username, clusterconfig.Password, kvaddress)
	FailTestIfError(err, "Error in ChangeIndexerSettings", t)
}
コード例 #3
0
ファイル: mutations_test.go プロジェクト: jchris/indexing
// Test with large number of mutations
func TestLargeMutations(t *testing.T) {
	log.Printf("In TestLargeMutations()")
	var index1 = "indexmut_1"
	var index2 = "indexmut_2"
	var bucketName = "default"
	var field1 = "company"
	var field2 = "gender"

	e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	FailTestIfError(e, "Error in DropAllSecondaryIndexes", t)

	docsToCreate := generateDocs(20000, "users.prod")
	UpdateKVDocs(docsToCreate, docs)
	kv.SetKeyValues(docsToCreate, "default", "", clusterconfig.KVAddress)

	err := secondaryindex.CreateSecondaryIndex(index1, bucketName, indexManagementAddress, "", []string{field1}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	docScanResults := datautility.ExpectedScanAllResponse(docs, field1)
	scanResults, err := secondaryindex.ScanAll(index1, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan", t)
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)
	log.Printf("Len of expected and actual scan results are :  %d and %d", len(docScanResults), len(scanResults))

	for i := 0; i <= 10; i++ {
		log.Printf("ITERATION %v\n", i)

		docsToCreate = generateDocs(10000, "users.prod")
		UpdateKVDocs(docsToCreate, docs)
		kv.SetKeyValues(docsToCreate, "default", "", clusterconfig.KVAddress)

		err := secondaryindex.CreateSecondaryIndex(index2, bucketName, indexManagementAddress, "", []string{field2}, false, nil, true, defaultIndexActiveTimeout, nil)
		FailTestIfError(err, "Error in creating the index", t)

		docScanResults = datautility.ExpectedScanAllResponse(docs, field1)
		scanResults, err = secondaryindex.ScanAll(index1, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil)
		FailTestIfError(err, "Error in scan", t)
		err = tv.Validate(docScanResults, scanResults)
		FailTestIfError(err, "Error in scan result validation", t)
		log.Printf("Len of expected and actual scan results are :  %d and %d", len(docScanResults), len(scanResults))

		docScanResults = datautility.ExpectedScanAllResponse(docs, field2)
		scanResults, err = secondaryindex.ScanAll(index2, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil)
		FailTestIfError(err, "Error in scan", t)
		err = tv.Validate(docScanResults, scanResults)
		FailTestIfError(err, "Error in scan result validation", t)
		log.Printf("Len of expected and actual scan results are :  %d and %d", len(docScanResults), len(scanResults))

		err = secondaryindex.DropSecondaryIndex(index2, bucketName, indexManagementAddress)
		FailTestIfError(err, "Error in drop index", t)
	}
}
コード例 #4
0
ファイル: setd_misc_test.go プロジェクト: jchris/indexing
func TestUpdateMutations_AddField(t *testing.T) {
	log.Printf("In TestUpdateMutations_AddField()")

	var bucketName = "default"
	var indexName = "index_newField"
	var field = "newField"

	docsToCreate := generateDocs(1000, "users.prod")
	UpdateKVDocs(docsToCreate, docs)

	seed++
	log.Printf("Setting JSON docs in KV")
	kvutility.SetKeyValues(docsToCreate, "default", "", clusterconfig.KVAddress)

	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{field}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	docScanResults := datautility.ExpectedScanAllResponse(docs, field)
	scanResults, err := secondaryindex.ScanAll(indexName, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan", t)
	log.Printf("Count of scan results before add field mutations:  %d", len(scanResults))
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)

	// Create mutations with add fields
	AddFieldMutations(300, field) // Update documents by adding the indexed field

	docScanResults = datautility.ExpectedScanAllResponse(docs, field)
	scanResults, err = secondaryindex.ScanAll(indexName, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan", t)
	log.Printf("Count of scan results after add field mutations:  %d", len(scanResults))
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)
}
コード例 #5
0
ファイル: setd_misc_test.go プロジェクト: jchris/indexing
// Test case for testing primary key values with longest length possible
func TestLargePrimaryKeyLength(t *testing.T) {
	log.Printf("In TestLargePrimaryKeyLength()")

	indexName := "index_LongPrimaryField"
	bucketName := "default"

	e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	FailTestIfError(e, "Error in DropAllSecondaryIndexes", t)

	largePrimaryKeyDocs := generateLargePrimaryKeyDocs(1000, "docid")
	seed++
	log.Printf("Setting JSON docs in KV")
	kvutility.SetKeyValues(largePrimaryKeyDocs, "default", "", clusterconfig.KVAddress)

	// Create a primary index
	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", nil, true, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	scanResults, err := secondaryindex.ScanAll(indexName, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan", t)
	if len(scanResults) != len(docs) {
		log.Printf("Len of scanResults is incorrect. Expected and Actual are:  %d and %d", len(docs), len(scanResults))
		err = errors.New("Len of scanResults is incorrect.")
		FailTestIfError(err, "Len of scanResults is incorrect", t)
	}

	log.Printf("Lengths of num of docs and scanResults are:  %d and %d", len(docs), len(scanResults))
	log.Printf("End: Length of kv docs is %d", len(docs))
}
コード例 #6
0
ファイル: setd_misc_test.go プロジェクト: jchris/indexing
// Test case for testing secondary key field values as very huge
func TestLargeSecondaryKeyLength(t *testing.T) {
	log.Printf("In TestLargeSecondaryKeyLength()")

	field := "LongSecField"
	indexName := "index_LongSecField"
	bucketName := "default"

	e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	FailTestIfError(e, "Error in DropAllSecondaryIndexes", t)

	largeKeyDocs := generateLargeSecondayKeyDocs(1000, field)
	seed++
	log.Printf("Setting JSON docs in KV")
	kvutility.SetKeyValues(largeKeyDocs, "default", "", clusterconfig.KVAddress)

	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{field}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	docScanResults := datautility.ExpectedScanAllResponse(docs, field)
	scanResults, err := secondaryindex.ScanAll(indexName, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan", t)
	log.Printf("ScanAll: Lengths of expected and actual scan results are:  %d and %d", len(docScanResults), len(scanResults))
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)

	docScanResults = datautility.ExpectedScanResponse_string(docs, field, "A", "zzzz", 3)
	scanResults, err = secondaryindex.Range(indexName, bucketName, indexScanAddress, []interface{}{"A"}, []interface{}{"zzzz"}, 3, true, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan", t)
	log.Printf("Range: Lengths of expected and actual scan results are:  %d and %d", len(docScanResults), len(scanResults))
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)
	log.Printf("End: Length of kv docs is %d", len(docs))
}
コード例 #7
0
ファイル: setd_misc_test.go プロジェクト: jchris/indexing
func TestMixedDatatypesScanAll(t *testing.T) {
	log.Printf("In TestMixedDatatypesScanAll()")
	log.Printf("Before test begin: Length of kv docs is %d", len(docs))
	field := "md_street"
	indexName := "index_mixeddt"
	bucketName := "default"

	e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	FailTestIfError(e, "Error in DropAllSecondaryIndexes", t)

	docsToCreate := generateJSONSMixedDatatype(1000, "md_street")
	log.Printf("After generate docs: Length of kv docs is %d", len(docs))
	seed++
	log.Printf("Setting mixed datatypes JSON docs in KV")
	kvutility.SetKeyValues(docsToCreate, "default", "", clusterconfig.KVAddress)

	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{field}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	docScanResults := datautility.ExpectedScanAllResponse(docs, field)
	scanResults, err := secondaryindex.ScanAll(indexName, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan", t)
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)
	log.Printf("Lengths of expected and actual scan results are:  %d and %d", len(docScanResults), len(scanResults))
	log.Printf("End: Length of kv docs is %d", len(docs))
}
コード例 #8
0
func TestIndexNameValidation(t *testing.T) {
	log.Printf("In TestIndexNameValidation()")

	var bucketName = "default"
	var validIndexName = "#primary-Index_test"
	var invalidIndexName = "ÌñÐÉx&(abc_%"
	var field = "balance"

	docsToCreate := generateDocs(1000, "users.prod")
	UpdateKVDocs(docsToCreate, docs)

	seed++
	log.Printf("Setting JSON docs in KV")
	kvutility.SetKeyValues(docsToCreate, "default", "", clusterconfig.KVAddress)

	err := secondaryindex.CreateSecondaryIndex(invalidIndexName, bucketName, indexManagementAddress, "", []string{field}, false, nil, true, defaultIndexActiveTimeout, nil)
	errMsg := "Expected index name validation error for index " + invalidIndexName
	FailTestIfNoError(err, errMsg, t)
	log.Printf("Creation of index with invalid name %v failed as expected", invalidIndexName)

	err = secondaryindex.CreateSecondaryIndex(validIndexName, bucketName, indexManagementAddress, "", []string{field}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	docScanResults := datautility.ExpectedScanResponse_string(docs, field, "$4", "$7", 3)
	scanResults, err := secondaryindex.Range(validIndexName, bucketName, indexScanAddress, []interface{}{"$4"}, []interface{}{"$7"}, 3, true, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan", t)
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)
}
コード例 #9
0
ファイル: concurrency_test.go プロジェクト: jchris/indexing
func TestRangeWithConcurrentDelMuts(t *testing.T) {
	log.Printf("In TestRangeWithConcurrentDelMuts()")
	var wg sync.WaitGroup
	prodfile = filepath.Join(proddir, "test.prod")
	// secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)

	log.Printf("Generating JSON docs")
	kvdocs = GenerateJsons(30000, seed, prodfile, bagdir)
	seed++

	log.Printf("Setting initial JSON docs in KV")
	kv.SetKeyValues(kvdocs, "default", "", clusterconfig.KVAddress)

	var indexName = "index_company"
	var bucketName = "default"

	log.Printf("Creating a 2i")
	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"company"}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	wg.Add(2)
	go DeleteDocsForDuration(&wg, 120)
	go RangeScanForDuration_ltr("Thread 1: ", &wg, 120, t, indexName, bucketName, indexScanAddress)
	wg.Wait()
}
コード例 #10
0
ファイル: setd_misc_test.go プロジェクト: jchris/indexing
func DataTypeChangeMutations_BoolToString(num int, field string) {
	i := 0
	keys := make(tc.KeyValues)
	for key, value := range docs {
		keys[key] = value
		i++
		if i == num {
			break
		}
	}

	keysToBeUpdated := make(tc.KeyValues)
	for key, value := range keys {
		json := value.(map[string]interface{})
		if json[field] == true {
			json[field] = "true"
		} else {
			json[field] = "false"
		}
		docs[key] = json
		keysToBeUpdated[key] = json
	}

	kvutility.SetKeyValues(keysToBeUpdated, "default", "", clusterconfig.KVAddress)
}
コード例 #11
0
func TestFieldsWithSpecialCharacters(t *testing.T) {
	log.Printf("In TestFieldsWithSpecialCharacters()")

	var bucketName = "default"
	var indexName = "index_specialchar"
	var field = "splfield"

	docsToCreate := generateDocsWithSpecialCharacters(1000, "users.prod", field)
	UpdateKVDocs(docsToCreate, docs)
	var valueToLookup string
	for _, v := range docsToCreate {
		json := v.(map[string]interface{})
		valueToLookup = json[field].(string)
		break
	}

	kvutility.SetKeyValues(docsToCreate, bucketName, "", clusterconfig.KVAddress)

	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{field}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	log.Printf("Looking up for value %v", valueToLookup)
	docScanResults := datautility.ExpectedScanResponse_string(docs, field, valueToLookup, valueToLookup, 3)
	scanResults, err := secondaryindex.Lookup(indexName, bucketName, indexScanAddress, []interface{}{valueToLookup}, true, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan", t)
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)
}
コード例 #12
0
func TestLargeDocumentSize(t *testing.T) {
	log.Printf("In TestLargeDocumentSize()")

	u, _ := user.Current()
	datapath := filepath.Join(u.HomeDir, "testdata/TwitterFeed1.txt.gz")
	tc.DownloadDataFile(tc.IndexTypesTwitterFeed1JSONDataS3, datapath, true)
	largeDocs := datautility.LoadJSONFromCompressedFile(datapath, "id_str")
	UpdateKVDocs(largeDocs, docs)

	log.Printf("Length of docs and largeDocs = %d and %d", len(docs), len(largeDocs))

	bucketName := "default"
	index1 := "index_userscreenname"

	kvutility.SetKeyValues(largeDocs, bucketName, "", clusterconfig.KVAddress)

	err := secondaryindex.CreateSecondaryIndex(index1, bucketName, indexManagementAddress, "", []string{"`user`.screen_name"}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	docScanResults := datautility.ExpectedScanAllResponse(docs, "user.screen_name")
	scanResults, err := secondaryindex.ScanAll(index1, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan 1: ", t)
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan 1:  result validation", t)
}
コード例 #13
0
ファイル: mutations_test.go プロジェクト: jchris/indexing
func UpdateDocs(num int) {
	i := 0

	// Pick some docs from mut_docs
	keysFromMutDocs := make(tc.KeyValues)
	for key, value := range mut_docs {
		keysFromMutDocs[key] = value
		i++
		if i == num {
			break
		}
	}
	// and , Add them to docs
	keysToBeSet := make(tc.KeyValues)
	for _, value := range keysFromMutDocs {
		n := randomNum(0, float64(len(docs)-1))
		i = 0
		var k string
		for k, _ = range docs {
			if i == n {
				break
			}
			i++
		}
		docs[k] = value
		keysToBeSet[k] = value
	}
	log.Printf("Num of keysFromMutDocs: %d", len(keysFromMutDocs))
	log.Printf("Updating number of documents: %d", len(keysToBeSet))
	kv.SetKeyValues(keysToBeSet, "default", "", clusterconfig.KVAddress)
}
コード例 #14
0
ファイル: concurrency_test.go プロジェクト: jchris/indexing
func CreateDeleteDocsForDuration(wg *sync.WaitGroup, seconds float64) {
	log.Printf("CreateDocs:: Creating mutations")
	defer wg.Done()

	start := time.Now()
	for {
		elapsed := time.Since(start)
		if elapsed.Seconds() >= seconds {
			break
		}

		docsToBeCreated := GenerateJsons(100, seed, prodfile, bagdir)
		seed++
		kv.SetKeyValues(docsToBeCreated, "default", "", clusterconfig.KVAddress)
		for key, value := range docsToBeCreated {
			kvdocs[key] = value
		}
		log.Printf("CreateDocs:: Len of kvdocs: %v", len(kvdocs))

		i := 0
		keysToBeDeleted := make(tc.KeyValues)
		for key, value := range kvdocs {
			keysToBeDeleted[key] = value
			i++
			if i == 5 {
				break
			}
		}
		kv.DeleteKeys(keysToBeDeleted, "default", "", clusterconfig.KVAddress)
		// Update docs object with deleted keys
		for key, _ := range keysToBeDeleted {
			delete(kvdocs, key)
		}
	}
}
コード例 #15
0
ファイル: concurrency_test.go プロジェクト: jchris/indexing
func TestMutationsWithMultipleIndexBuilds(t *testing.T) {
	log.Printf("In TestMutationsWithMultipleIndexBuilds()")
	prodfile = filepath.Join(proddir, "test.prod")
	secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)

	log.Printf("Generating JSON docs")
	kvdocs = GenerateJsons(100000, seed, prodfile, bagdir)
	seed++

	log.Printf("Setting initial JSON docs in KV")
	kv.SetKeyValues(kvdocs, "default", "", clusterconfig.KVAddress)

	var index_p = "index_primary"
	var bucketName = "default"
	indexes := []string{"index_company", "index_age", "index_firstname", "index_lastname"}
	fields := []string{"company", "age", "`first-name`", "`last-name`"}

	// Create a primary index
	err := secondaryindex.CreateSecondaryIndex(index_p, bucketName, indexManagementAddress, "", nil, true, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	log.Printf("Creating multiple indexes in deferred mode")
	for i := range indexes {
		e := secondaryindex.CreateSecondaryIndexAsync(indexes[i], bucketName, indexManagementAddress, "", []string{fields[i]}, false, []byte("{\"defer_build\": true}"), true, nil)
		FailTestIfError(e, "Error in creating the index", t)
	}

	log.Printf("Build Indexes and wait for indexes to become active: %v", indexes)
	err = secondaryindex.BuildIndexes(indexes, bucketName, indexManagementAddress, 2000)
	FailTestIfError(err, "Error in deferred index build", t)
}
コード例 #16
0
ファイル: perf_test.go プロジェクト: jchris/indexing
func SkipTestPerfRangeWithoutMutations_1M(t *testing.T) {
	log.Printf("In TestPerfQueryWithoutMutations_1M()")
	e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	FailTestIfError(e, "Error in DropAllSecondaryIndexes", t)

	prodfile := "../../../../../prataprc/monster/prods/test.prod"
	bagdir := "../../../../../prataprc/monster/bags/"
	log.Printf("Generating JSON docs")
	count := 1000000
	keyValues := GenerateJsons(count, 1, prodfile, bagdir)
	log.Printf("Setting JSON docs in KV")
	kv.SetKeyValues(keyValues, "default", "", clusterconfig.KVAddress)

	var indexName = "index_company"
	var bucketName = "default"

	log.Printf("Creating a 2i")

	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"company"}, false, nil, true, 1500, nil)
	FailTestIfError(err, "Error in creating the index", t)

	start := time.Now()
	scanResults, err := secondaryindex.Range(indexName, bucketName, indexScanAddress, []interface{}{"E"}, []interface{}{"N"}, 3, true, defaultlimit, c.AnyConsistency, nil)
	elapsed := time.Since(start)
	log.Printf("RangeScan:: Len of scanResults is: %d", len(scanResults))
	log.Printf("Range Scan of %d user documents took %s\n", count, elapsed)
}
コード例 #17
0
func TestSameFieldNameAtDifferentLevels(t *testing.T) {
	log.Printf("In TestSameFieldNameAtDifferentLevels()")

	var bucketName = "default"
	var indexName = "cityindex"
	var field = "city"

	docsToCreate := generateDocs(1000, "users.prod")
	UpdateKVDocs(docsToCreate, docs)
	docsToUpload := make(tc.KeyValues)

	for k, v := range docsToCreate {
		json := v.(map[string]interface{})
		address := json["address"].(map[string]interface{})
		city := address["city"].(string)
		json[field] = city
		address["city"] = "ThisIsNestedCity " + city
		docsToUpload[k] = json
	}

	seed++
	log.Printf("Setting JSON docs in KV")
	kvutility.SetKeyValues(docsToUpload, "default", "", clusterconfig.KVAddress)

	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{field}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	docScanResults := datautility.ExpectedScanResponse_string(docs, field, "A", "K", 3)
	scanResults, err := secondaryindex.Range(indexName, bucketName, indexScanAddress, []interface{}{"A"}, []interface{}{"K"}, 3, true, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan", t)
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)
}
コード例 #18
0
ファイル: perf_test.go プロジェクト: jchris/indexing
func SkipTestPerfPrimaryIndexBuild_SimpleJson(t *testing.T) {
	log.Printf("In TestPerfInitialIndexBuild()")
	e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	FailTestIfError(e, "Error in DropAllSecondaryIndexes", t)

	prodfile = filepath.Join(proddir, "test.prod")
	log.Printf("Generating JSON docs")
	count := 100000000
	keyValues := GenerateJsons(count, 1, prodfile, bagdir)
	log.Printf("Setting JSON docs in KV")
	kv.SetKeyValues(keyValues, "default", "", clusterconfig.KVAddress)

	var indexName = "index_primary"
	var bucketName = "default"

	log.Printf("Creating a 2i")
	start := time.Now()
	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", nil, true, nil, true, 1500, nil)
	FailTestIfError(err, "Error in creating the index", t)
	elapsed := time.Since(start)
	log.Printf("Index build of %d user documents took %s\n", count, elapsed)
	scanResults, err := secondaryindex.ScanAll(indexName, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan", t)
	if len(scanResults) != len(keyValues) {
		log.Printf("Len of scanResults is incorrect. Expected and Actual are: %d and %d", len(keyValues), len(scanResults))
		err = errors.New("Len of scanResults is incorrect.")
	}
	FailTestIfError(err, "Len of scanResults is incorrect", t)
}
コード例 #19
0
ファイル: perf_test.go プロジェクト: jchris/indexing
func SkipTestPerfRangeWithConcurrentKVMuts(t *testing.T) {
	log.Printf("In TestPerfRangeWithConcurrentKVMuts()")
	var wg sync.WaitGroup
	prodfile = "../../../../../prataprc/monster/prods/test.prod"
	bagdir = "../../../../../prataprc/monster/bags/"
	e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	FailTestIfError(e, "Error in DropAllSecondaryIndexes", t)

	log.Printf("Generating JSON docs")
	kvdocs = GenerateJsons(1000, seed, prodfile, bagdir)
	seed++

	log.Printf("Setting initial JSON docs in KV")
	kv.SetKeyValues(kvdocs, "default", "", clusterconfig.KVAddress)

	var indexName = "index_company"
	var bucketName = "default"

	log.Printf("Creating a 2i")
	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"company"}, false, nil, true, 1500, nil)
	FailTestIfError(err, "Error in creating the index", t)

	wg.Add(2)
	go CreateDeleteDocsForDuration(&wg, 60)
	go RangeScanForDuration_ltr("Range Thread: ", &wg, 60, t, indexName, bucketName, indexScanAddress)
	wg.Wait()
}
コード例 #20
0
ファイル: setd_misc_test.go プロジェクト: jchris/indexing
func TestMixedDatatypesRange_Float(t *testing.T) {
	log.Printf("In TestMixedDatatypesRange_Float()")

	field := "mixed_field"
	indexName := "index_mixeddt"
	bucketName := "default"

	e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	FailTestIfError(e, "Error in DropAllSecondaryIndexes", t)

	docsToCreate := generateJSONSMixedDatatype(1000, field)
	seed++
	log.Printf("Setting mixed datatypes JSON docs in KV")
	kvutility.SetKeyValues(docsToCreate, "default", "", clusterconfig.KVAddress)

	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{field}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	docScanResults := datautility.ExpectedScanResponse_float64(docs, field, 100, 1000, 3)
	scanResults, err := secondaryindex.Range(indexName, bucketName, indexScanAddress, []interface{}{100}, []interface{}{1000}, 3, true, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan", t)
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)
	log.Printf("Lengths of expected and actual scan results are:  %d and %d", len(docScanResults), len(scanResults))

	docScanResults = datautility.ExpectedScanResponse_float64(docs, field, 1, 100, 2)
	scanResults, err = secondaryindex.Range(indexName, bucketName, indexScanAddress, []interface{}{1}, []interface{}{100}, 2, true, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan", t)
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)
	log.Printf("Lengths of expected and actual scan results are:  %d and %d", len(docScanResults), len(scanResults))
	log.Printf("Length of kv docs is %d", len(docs))
}
コード例 #21
0
ファイル: concurrency_test.go プロジェクト: jchris/indexing
func TestConcurrentScans_MultipleIndexes(t *testing.T) {
	log.Printf("In TestConcurrentScans_MultipleIndexes()")
	var wg sync.WaitGroup
	prodfile = filepath.Join(proddir, "test.prod")
	// secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)

	log.Printf("Generating JSON docs")
	kvdocs = GenerateJsons(100000, seed, prodfile, bagdir)
	seed++

	log.Printf("Setting initial JSON docs in KV")
	kv.SetKeyValues(kvdocs, "default", "", clusterconfig.KVAddress)

	var index1 = "index_company"
	var index2 = "index_age"
	var index3 = "index_firstname"
	var bucketName = "default"

	log.Printf("Creating multiple indexes")
	err := secondaryindex.CreateSecondaryIndex(index1, bucketName, indexManagementAddress, "", []string{"company"}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	err = secondaryindex.CreateSecondaryIndex(index2, bucketName, indexManagementAddress, "", []string{"age"}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	err = secondaryindex.CreateSecondaryIndex(index3, bucketName, indexManagementAddress, "", []string{"`first-name`"}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	wg.Add(3)
	go RangeScanForDuration_ltr("Thread 1: ", &wg, 120, t, index1, bucketName, indexScanAddress)
	go RangeScanForDuration_num("Thread 2: ", &wg, 120, t, index2, bucketName, indexScanAddress)
	go RangeScanForDuration_ltr("Thread 3: ", &wg, 120, t, index3, bucketName, indexScanAddress)
	wg.Wait()
}
コード例 #22
0
func TestSameIndexNameInTwoBuckets(t *testing.T) {
	log.Printf("In TestSameIndexNameInTwoBuckets()")

	e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	FailTestIfError(e, "Error in DropAllSecondaryIndexes", t)

	numOfBuckets := 2
	indexName := "b_idx"
	indexFields := [...]string{"age", "address.city"}
	bucketNames := [...]string{"default", "buck2"}
	proxyPorts := [...]string{"11212", "11213"}
	var bucketDocs [2]tc.KeyValues

	// Update default bucket ram to 256
	// Create two more buckets with ram 256 each
	// Add different docs to all 3 buckets
	// Create index on each of them
	// Query the indexes

	kvutility.FlushBucket(bucketNames[0], "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	kvutility.EditBucket(bucketNames[0], "", clusterconfig.Username, clusterconfig.Password, kvaddress, "256")

	for i := 1; i < numOfBuckets; i++ {
		kvutility.CreateBucket(bucketNames[i], "sasl", "", clusterconfig.Username, clusterconfig.Password, kvaddress, "256", proxyPorts[i])
	}
	time.Sleep(30 * time.Second)

	log.Printf("Generating docs and Populating all the buckets")
	for i := 0; i < numOfBuckets; i++ {
		bucketDocs[i] = generateDocs(1000, "users.prod")
		kvutility.SetKeyValues(bucketDocs[i], bucketNames[i], "", clusterconfig.KVAddress)
		err := secondaryindex.CreateSecondaryIndex(indexName, bucketNames[i], indexManagementAddress, "", []string{indexFields[i]}, false, nil, true, defaultIndexActiveTimeout, nil)
		FailTestIfError(err, "Error in creating the index", t)
	}
	time.Sleep(3 * time.Second)

	// Scan index of first bucket
	docScanResults := datautility.ExpectedScanResponse_float64(bucketDocs[0], indexFields[0], 30, 50, 1)
	scanResults, err := secondaryindex.Range(indexName, bucketNames[0], indexScanAddress, []interface{}{30}, []interface{}{50}, 1, true, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan 1", t)
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)

	// Scan index of second bucket
	docScanResults = datautility.ExpectedScanResponse_string(bucketDocs[1], indexFields[1], "F", "Q", 2)
	scanResults, err = secondaryindex.Range(indexName, bucketNames[1], indexScanAddress, []interface{}{"F"}, []interface{}{"Q"}, 2, true, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan 2", t)
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)

	kvutility.EditBucket(bucketNames[0], "", clusterconfig.Username, clusterconfig.Password, kvaddress, "512")
	kvutility.DeleteBucket(bucketNames[1], "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	time.Sleep(30 * time.Second) // Sleep after bucket create or delete

	tc.ClearMap(docs)
	UpdateKVDocs(bucketDocs[0], docs)
}
コード例 #23
0
ファイル: common_test.go プロジェクト: jchris/indexing
func addDocIfNotPresentInKV(docKey string) {
	if _, present := docs[docKey]; present == false {
		keysToBeSet := make(tc.KeyValues)
		keysToBeSet[docKey] = mut_docs[docKey]
		kvutility.SetKeyValues(keysToBeSet, "default", "", clusterconfig.KVAddress)
		// Update docs object with newly added keys and remove those keys from mut_docs
		docs[docKey] = mut_docs[docKey]
		delete(mut_docs, docKey)
	}
}
コード例 #24
0
ファイル: setd_misc_test.go プロジェクト: jchris/indexing
// After bucket delete:-
// 1) query for old index before loading bucket
// 2) query for old index after loading bucket
// 3) create new indexes and query
// 4) list indexes: should list only new indexes
func TestBucketDefaultDelete(t *testing.T) {
	kvutility.DeleteBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	time.Sleep(30 * time.Second) // Sleep after bucket create or delete
	kvutility.CreateBucket("default", "none", "", clusterconfig.Username, clusterconfig.Password, kvaddress, "256", "11212")
	time.Sleep(30 * time.Second) // Sleep after bucket create or delete

	tc.ClearMap(docs)
	tc.ClearMap(mut_docs)
	docs = datautility.LoadJSONFromCompressedFile(dataFilePath, "docid")
	mut_docs = datautility.LoadJSONFromCompressedFile(mutationFilePath, "docid")
	log.Printf("Populating the default bucket")
	kvutility.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)
	time.Sleep(2 * time.Second)

	var indexName = "index_isActive"
	var bucketName = "default"

	scanResults, err := secondaryindex.Lookup(indexName, bucketName, indexScanAddress, []interface{}{"BIOSPAN"}, true, defaultlimit, c.SessionConsistency, nil)
	if err == nil {
		log.Printf("Scan did not fail as expected. Got scanresults: %v\n", scanResults)
		e := errors.New("Scan did not fail as expected after bucket delete")
		FailTestIfError(e, "Error in TestBucketDefaultDelete", t)
	} else {
		log.Printf("Scan failed as expected with error: %v", err)
	}

	log.Printf("Populating the default bucket after it was deleted")
	kvutility.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)

	err = secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"company"}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	docScanResults := datautility.ExpectedScanResponse_string(docs, "company", "BIOSPAN", "BIOSPAN", 3)
	scanResults, err = secondaryindex.Lookup(indexName, bucketName, indexScanAddress, []interface{}{"BIOSPAN"}, true, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan", t)
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)
	// todo: list the index and confirm there is only index created
}
コード例 #25
0
ファイル: common_test.go プロジェクト: jchris/indexing
func init() {
	log.Printf("In init()")
	logging.SetLogLevel(logging.Warn)

	var configpath string
	flag.StringVar(&configpath, "cbconfig", "../config/clusterrun_conf.json", "Path of the configuration file with data about Couchbase Cluster")
	flag.Parse()
	clusterconfig = tc.GetClusterConfFromFile(configpath)
	kvaddress = clusterconfig.KVAddress
	indexManagementAddress = clusterconfig.KVAddress
	indexScanAddress = clusterconfig.KVAddress
	seed = 1
	proddir, bagdir = tc.FetchMonsterToolPath()

	// setup cbauth
	if _, err := cbauth.InternalRetryDefaultInit(kvaddress, clusterconfig.Username, clusterconfig.Password); err != nil {
		log.Fatalf("Failed to initialize cbauth: %s", err)
	}
	secondaryindex.CheckCollation = true
	e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	tc.HandleError(e, "Error in DropAllSecondaryIndexes")

	time.Sleep(5 * time.Second)
	// Working with Users10k and Users_mut dataset.
	u, _ := user.Current()
	dataFilePath = filepath.Join(u.HomeDir, "testdata/Users10k.txt.gz")
	mutationFilePath = filepath.Join(u.HomeDir, "testdata/Users_mut.txt.gz")
	tc.DownloadDataFile(tc.IndexTypesStaticJSONDataS3, dataFilePath, true)
	tc.DownloadDataFile(tc.IndexTypesMutationJSONDataS3, mutationFilePath, true)
	docs = datautility.LoadJSONFromCompressedFile(dataFilePath, "docid")
	mut_docs = datautility.LoadJSONFromCompressedFile(mutationFilePath, "docid")
	log.Printf("Emptying the default bucket")
	kvutility.EnableBucketFlush("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	kvutility.FlushBucket("default", "", clusterconfig.Username, clusterconfig.Password, kvaddress)
	time.Sleep(5 * time.Second)

	log.Printf("Create Index On the empty default Bucket()")
	var indexName = "index_eyeColor"
	var bucketName = "default"

	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"eyeColor"}, false, nil, true, defaultIndexActiveTimeout, nil)
	tc.HandleError(err, "Error in creating the index")

	// Populate the bucket now
	log.Printf("Populating the default bucket")
	kvutility.SetKeyValues(docs, "default", "", clusterconfig.KVAddress)
}
コード例 #26
0
ファイル: mutations_test.go プロジェクト: jchris/indexing
func CreateDocs(num int) {
	i := 0
	keysToBeSet := make(tc.KeyValues)
	for key, value := range mut_docs {
		keysToBeSet[key] = value
		i++
		if i == num {
			break
		}
	}
	kv.SetKeyValues(keysToBeSet, "default", "", clusterconfig.KVAddress)
	// Update docs object with newly added keys and remove those keys from mut_docs
	for key, value := range keysToBeSet {
		docs[key] = value
		delete(mut_docs, key)
	}
}
コード例 #27
0
ファイル: setd_misc_test.go プロジェクト: jchris/indexing
func TestUpdateMutations_DataTypeChange(t *testing.T) {
	log.Printf("In TestUpdateMutations_DataTypeChange()")

	var bucketName = "default"
	var indexName = "index_isUserActive"
	var field = "isActive"

	docsToCreate := generateDocs(1000, "users.prod")
	UpdateKVDocs(docsToCreate, docs)

	seed++
	log.Printf("Setting JSON docs in KV")
	kvutility.SetKeyValues(docsToCreate, "default", "", clusterconfig.KVAddress)
	time.Sleep(2 * time.Second)

	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{field}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)

	docScanResults := datautility.ExpectedScanAllResponse(docs, field)
	scanResults, err := secondaryindex.ScanAll(indexName, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil)
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)

	// Create mutations with delete fields
	DataTypeChangeMutations_BoolToString(200, field) // Update documents by changing datatype of the indexed field

	docScanResults = datautility.ExpectedScanAllResponse(docs, field)
	scanResults, err = secondaryindex.ScanAll(indexName, bucketName, indexScanAddress, defaultlimit, c.SessionConsistency, nil)
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)

	docScanResults = datautility.ExpectedScanResponse_string(docs, field, "true", "true", 3)
	scanResults, err = secondaryindex.Lookup(indexName, bucketName, indexScanAddress, []interface{}{"true"}, true, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan", t)
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)

	docScanResults = datautility.ExpectedScanResponse_string(docs, field, "false", "false", 3)
	scanResults, err = secondaryindex.Lookup(indexName, bucketName, indexScanAddress, []interface{}{"false"}, true, defaultlimit, c.SessionConsistency, nil)
	FailTestIfError(err, "Error in scan", t)
	err = tv.Validate(docScanResults, scanResults)
	FailTestIfError(err, "Error in scan result validation", t)
}
コード例 #28
0
ファイル: concurrency_test.go プロジェクト: jchris/indexing
func CreateDocsForDuration(wg *sync.WaitGroup, seconds float64) {
	log.Printf("CreateDocs:: Creating mutations")
	defer wg.Done()
	start := time.Now()
	for {
		elapsed := time.Since(start)
		if elapsed.Seconds() >= seconds {
			break
		}

		docsToBeCreated := GenerateJsons(100, seed, prodfile, bagdir)
		seed++
		kv.SetKeyValues(docsToBeCreated, "default", "", clusterconfig.KVAddress)
		for key, value := range docsToBeCreated {
			kvdocs[key] = value
		}
		// log.Printf("CreateDocs:: Len of kvdocs", len(kvdocs))
	}
}
コード例 #29
0
ファイル: setd_misc_test.go プロジェクト: jchris/indexing
func DeleteFieldMutations(num int, field string) {
	i := 0
	keys := make(tc.KeyValues)
	for key, value := range docs {
		keys[key] = value
		i++
		if i == num {
			break
		}
	}

	keysToBeUpdated := make(tc.KeyValues)
	for key, value := range keys {
		json := value.(map[string]interface{})
		delete(json, field)
		docs[key] = json
		keysToBeUpdated[key] = json
	}

	kvutility.SetKeyValues(keysToBeUpdated, "default", "", clusterconfig.KVAddress)
}
コード例 #30
0
ファイル: concurrency_test.go プロジェクト: jchris/indexing
func SkipTestSequentialRangeScans(t *testing.T) {
	log.Printf("In TestSequentialRangeScans()")
	prodfile = filepath.Join(proddir, "test.prod")
	e := secondaryindex.DropAllSecondaryIndexes(indexManagementAddress)
	FailTestIfError(e, "Error in DropAllSecondaryIndexes", t)

	log.Printf("Generating JSON docs")
	kvdocs = GenerateJsons(1000, seed, prodfile, bagdir)
	seed++

	log.Printf("Setting initial JSON docs in KV")
	kv.SetKeyValues(kvdocs, "default", "", clusterconfig.KVAddress)

	var indexName = "index_company"
	var bucketName = "default"

	log.Printf("Creating a 2i")
	err := secondaryindex.CreateSecondaryIndex(indexName, bucketName, indexManagementAddress, "", []string{"company"}, false, nil, true, defaultIndexActiveTimeout, nil)
	FailTestIfError(err, "Error in creating the index", t)
	SequentialRangeScanForDuration(indexName, bucketName, 60, t)
}