Example #1
0
func runDeleteTest() {

	logging.Infof("**** Run Delete Test *******************************************")
	delete_test_status = make(map[uint64]*protobuf.Instance)

	cfg := common.SystemConfig.SectionConfig("indexer", true /*trim*/)
	cfg.Set("storage_dir", common.ConfigValue{"./data/", "metadata file path", "./"})
	os.MkdirAll("./data/", os.ModePerm)

	logging.Infof("***** Start TestStreamMgr ")
	/*
		var requestAddr = "localhost:9885"
		var leaderAddr = "localhost:9884"
	*/
	var config = "./config.json"

	logging.Infof("Start Index Manager")
	donech := make(chan bool)
	factory := new(deleteTestProjectorClientFactory)
	factory.donech = donech
	env := new(deleteTestProjectorClientEnv)
	admin := manager.NewProjectorAdmin(factory, env, nil)
	//mgr, err := manager.NewIndexManagerInternal(requestAddr, leaderAddr, config, admin)
	mgr, err := manager.NewIndexManagerInternal("localhost:9886", "localhost:"+manager.COORD_MAINT_STREAM_PORT, admin, cfg)
	if err != nil {
		util.TT.Fatal(err)
	}
	mgr.StartCoordinator(config)
	time.Sleep(time.Duration(3000) * time.Millisecond)

	logging.Infof("Delete Test Cleanup ...")
	cleanupStreamMgrDeleteTest(mgr)

	logging.Infof("***** Run Delete Test :  setup")
	ch := mgr.GetStabilityTimestampChannel(common.MAINT_STREAM)
	go runDeleteTestReceiver(ch, donech)

	logging.Infof("***** Run Delete Test :  add 3 indexes in 2 buckets to trigger topology change")
	addIndexForDeleteTest(mgr)

	logging.Infof("***** Run Delete Test :  delete index in 'Default' buckets to trigger topology change")
	deleteIndexForDeleteTest(mgr)

	logging.Infof("***** Run Delete Test :  wait for stability timestamp to arrive")
	<-donech

	logging.Infof("**** Delete Test Cleanup ...")
	cleanupStreamMgrDeleteTest(mgr)
	mgr.CleanupTopology()
	mgr.CleanupStabilityTimestamp()
	time.Sleep(time.Duration(1000) * time.Millisecond)

	logging.Infof("**** Stop TestStreamMgr. Tearing down ")
	mgr.Close()
	time.Sleep(time.Duration(1000) * time.Millisecond)

	logging.Infof("**** Finish Delete Test ***********************************************************")
}
Example #2
0
func runTimerTest() {

	logging.Infof("**** Run Timer Test **********************************************")

	cfg := common.SystemConfig.SectionConfig("indexer", true /*trim*/)
	cfg.Set("storage_dir", common.ConfigValue{"./data/", "metadata file path", "./"})
	os.MkdirAll("./data/", os.ModePerm)

	logging.Infof("***** Start TestStreamMgr ")
	/*
		var requestAddr = "localhost:9885"
		var leaderAddr = "localhost:9884"
	*/
	var config = "./config.json"

	logging.Infof("Start Index Manager")
	donech := make(chan bool)
	factory := new(timerTestProjectorClientFactory)
	factory.donech = donech
	env := new(timerTestProjectorClientEnv)
	admin := manager.NewProjectorAdmin(factory, env, nil)
	//mgr, err := manager.NewIndexManagerInternal(requestAddr, leaderAddr, config, admin)
	mgr, err := manager.NewIndexManagerInternal("localhost:9886", "localhost:"+manager.COORD_MAINT_STREAM_PORT, admin, cfg)
	if err != nil {
		util.TT.Fatal(err)
	}
	mgr.StartCoordinator(config)
	time.Sleep(time.Duration(3000) * time.Millisecond)

	mgr.SetTimestampPersistenceInterval(1)
	defer mgr.SetTimestampPersistenceInterval(manager.TIMESTAMP_PERSIST_INTERVAL)

	logging.Infof("Timer Test Cleanup ...")
	cleanupStreamMgrTimerTest(mgr)

	logging.Infof("***** Run timer Test ...")
	ch := mgr.GetStabilityTimestampChannel(common.MAINT_STREAM)
	go runTimerTestReceiver(mgr, ch, donech)

	logging.Infof("Setup data for Timer Test")
	changeTopologyForTimerTest(mgr)
	<-donech

	logging.Infof("**** Timer Test Cleanup ...")
	cleanupStreamMgrTimerTest(mgr)
	mgr.CleanupTopology()
	mgr.CleanupStabilityTimestamp()
	time.Sleep(time.Duration(1000) * time.Millisecond)

	logging.Infof("**** Stop TestStreamMgr. Tearing down ")
	mgr.Close()
	time.Sleep(time.Duration(1000) * time.Millisecond)

	logging.Infof("**** Finish Timer Test *************************************************")
}
Example #3
0
// For this test, use Index Defn Id from 100 - 110
func TestIndexManager(t *testing.T) {

	logging.SetLogLevel(logging.Trace)
	os.MkdirAll("./data/", os.ModePerm)

	cfg := common.SystemConfig.SectionConfig("indexer", true /*trim*/)
	cfg.Set("storage_dir", common.ConfigValue{"./data/", "metadata file path", "./"})

	logging.Infof("Start Index Manager *********************************************************")

	var msgAddr = "localhost:9884"
	factory := new(util.TestDefaultClientFactory)
	env := new(util.TestDefaultClientEnv)
	admin := manager.NewProjectorAdmin(factory, env, nil)
	mgr, err := manager.NewIndexManagerInternal(msgAddr, "localhost:"+manager.COORD_MAINT_STREAM_PORT, admin, cfg)
	if err != nil {
		t.Fatal(err)
	}
	defer mgr.Close()

	logging.Infof("Cleanup Test *********************************************************")

	cleanupTest(mgr, t)

	logging.Infof("Setup Initial Data *********************************************************")

	//setupInitialData(mgr, t)

	logging.Infof("Start Provider *********************************************************")

	var providerId = "TestMetadataProvider"
	provider, err := client.NewMetadataProvider(providerId)
	if err != nil {
		t.Fatal(err)
	}
	defer provider.Close()
	provider.WatchMetadata(msgAddr)

	logging.Infof("Test Iterator *********************************************************")

	runIterator(mgr, t, 0)

	plan := make(map[string]interface{})
	plan["nodes"] = []string{msgAddr}
	plan["defer_build"] = true
	newDefnId101, err := provider.CreateIndexWithPlan("manager_test_101", "Default", common.ForestDB,
		common.N1QL, "Testing", "TestingWhereExpr", []string{"Testing"}, false, plan)
	if err != nil {
		t.Fatal("Cannot create Index Defn 101 through MetadataProvider")
	}
	runIterator(mgr, t, 1)

	newDefnId102, err := provider.CreateIndexWithPlan("manager_test_102", "Default", common.ForestDB,
		common.N1QL, "Testing", "TestingWhereExpr", []string{"Testing"}, false, plan)
	if err != nil {
		t.Fatal("Cannot create Index Defn 102 through MetadataProvider")
	}
	runIterator(mgr, t, 2)

	logging.Infof("Cleanup Test *********************************************************")

	provider.UnwatchMetadata(msgAddr)
	cleanSingleIndex_managerTest(mgr, t, newDefnId101)
	cleanSingleIndex_managerTest(mgr, t, newDefnId102)
	time.Sleep(time.Duration(1000) * time.Millisecond)
}
// For this test, use Index Defn Id from 100 - 110
func TestMetadataProvider(t *testing.T) {

	logging.SetLogLevel(logging.Trace)

	cfg := common.SystemConfig.SectionConfig("indexer", true /*trim*/)
	cfg.Set("storage_dir", common.ConfigValue{"./data/", "metadata file path", "./"})
	os.MkdirAll("./data/", os.ModePerm)

	logging.Infof("Start Index Manager *********************************************************")

	var msgAddr = "localhost:9884"
	var httpAddr = "localhost:9885"
	factory := new(util.TestDefaultClientFactory)
	env := new(util.TestDefaultClientEnv)
	admin := manager.NewProjectorAdmin(factory, env, nil)
	addrPrv := util.NewFakeAddressProvider(msgAddr, httpAddr)
	mgr, err := manager.NewIndexManagerInternal(addrPrv, admin, cfg)
	if err != nil {
		t.Fatal(err)
	}
	defer mgr.Close()
	gMgr = mgr

	logging.Infof("Cleanup Test *********************************************************")

	cleanupTest(mgr, t)

	logging.Infof("Setup Initial Data *********************************************************")

	setupInitialData(mgr, t)

	logging.Infof("Start Provider *********************************************************")

	var providerId = "TestMetadataProvider"
	provider, err := client.NewMetadataProvider(providerId)
	if err != nil {
		t.Fatal(err)
	}
	defer provider.Close()
	provider.SetTimeout(int64(time.Second) * 15)
	indexerId, err := provider.WatchMetadata(msgAddr)
	if err != nil {
		t.Fatal(err)
	}

	// the gometa server is running in the same process as MetadataProvider (client).  So sleep to
	// make sure that the server has a chance to finish off initialization, since the client may
	// be ready, but the server is not.
	time.Sleep(time.Duration(1000) * time.Millisecond)

	logging.Infof("Verify Initial Data *********************************************************")

	meta := lookup(provider, common.IndexDefnId(100))
	if meta == nil {
		t.Fatal("Cannot find Index Defn 100 from MetadataProvider")
	}
	logging.Infof("found Index Defn 100")
	if len(meta.Instances) == 0 || meta.Instances[0].State != common.INDEX_STATE_READY {
		t.Fatal("Index Defn 100 state is not ready")
	}
	if meta.Instances[0].IndexerId != indexerId {
		t.Fatal("Index Defn 100 state is not ready")
	}

	meta = lookup(provider, common.IndexDefnId(101))
	if meta == nil {
		t.Fatal("Cannot find Index Defn 101 from MetadataProvider")
	}
	logging.Infof("found Index Defn 101")
	if len(meta.Instances) == 0 || meta.Instances[0].State != common.INDEX_STATE_READY {
		t.Fatal("Index Defn 101 state is not ready")
	}
	if meta.Instances[0].IndexerId != indexerId {
		t.Fatal("Index Defn 100 state is not ready")
	}

	logging.Infof("Change Data *********************************************************")

	notifier := &notifier{hasCreated: false, hasDeleted: false}
	mgr.RegisterNotifier(notifier)

	// Create Index with deployment plan (deferred)
	plan := make(map[string]interface{})
	plan["nodes"] = []interface{}{msgAddr}
	plan["defer_build"] = true
	newDefnId, err := provider.CreateIndexWithPlan("metadata_provider_test_102", "Default", common.ForestDB,
		common.N1QL, "Testing", "TestingWhereExpr", []string{"Testing"}, false, plan)
	if err != nil {
		t.Fatal("Cannot create Index Defn 102 through MetadataProvider" + err.Error())
	}
	input := make([]common.IndexDefnId, 1)
	input[0] = newDefnId
	if err := provider.BuildIndexes(input); err != nil {
		t.Fatal("Cannot build Index Defn : %v", err)
	}
	logging.Infof("done creating index 102")

	// Drop a seeded index (created during setup step)
	if err := provider.DropIndex(common.IndexDefnId(101)); err != nil {
		t.Fatal("Cannot drop Index Defn 101 through MetadataProvider")
	}
	logging.Infof("done dropping index 101")

	// Create Index (immediate).
	newDefnId2, err := provider.CreateIndexWithPlan("metadata_provider_test_103", "Default", common.ForestDB,
		common.N1QL, "Testing", "TestingWhereExpr", []string{"Testing"}, false, nil)
	if err != nil {
		t.Fatal("Cannot create Index Defn 103 through MetadataProvider")
	}
	logging.Infof("done creating index 103")

	// Update instance (set state to ACTIVE)
	if err := mgr.UpdateIndexInstance("Default", newDefnId2, common.INDEX_STATE_ACTIVE, common.StreamId(100), "", nil); err != nil {
		t.Fatal("Fail to update index instance")
	}
	logging.Infof("done updating index 103")

	// Update instance (set error string)
	if err := mgr.UpdateIndexInstance("Default", newDefnId2, common.INDEX_STATE_NIL, common.NIL_STREAM, "testing", nil); err != nil {
		t.Fatal("Fail to update index instance")
	}
	logging.Infof("done updating index 103")

	// Create Index (immediate).  This index is supposed to fail by OnIndexBuild()
	if _, err := provider.CreateIndexWithPlan("metadata_provider_test_104", "Default", common.ForestDB,
		common.N1QL, "Testing", "Testing", []string{"Testing"}, false, nil); err == nil {
		t.Fatal("Error does not propage for create Index Defn 104 through MetadataProvider")
	}
	logging.Infof("done creating index 104")

	logging.Infof("Verify Changed Data *********************************************************")

	if lookup(provider, common.IndexDefnId(100)) == nil {
		t.Fatal("Cannot find Index Defn 100 from MetadataProvider")
	}
	logging.Infof("found Index Defn 100")

	if lookup(provider, common.IndexDefnId(101)) != nil {
		t.Fatal("Found Deleted Index Defn 101 from MetadataProvider")
	}
	logging.Infof("cannot found deleted Index Defn 101")

	if meta = lookup(provider, newDefnId); meta == nil {
		t.Fatal(fmt.Sprintf("Cannot Found Index Defn %d from MetadataProvider", newDefnId))
	} else {
		logging.Infof("Found Index Defn %d", newDefnId)
		logging.Infof("meta.Instance %v", meta.Instances)
		if meta.Instances[0].IndexerId != indexerId {
			t.Fatal(fmt.Sprintf("Index Defn %v has incorrect endpoint", newDefnId))
		}
		if meta.Definition.WhereExpr != "TestingWhereExpr" {
			t.Fatal(fmt.Sprintf("WhereExpr is missing in Index Defn %v", newDefnId))
		}
		if meta.Instances[0].State != common.INDEX_STATE_INITIAL {
			t.Fatal(fmt.Sprintf("Index Defn %v has incorrect state", newDefnId))
		}
	}

	if meta = lookup(provider, newDefnId2); meta == nil {
		t.Fatal(fmt.Sprintf("Cannot Found Index Defn %d from MetadataProvider", newDefnId2))
	} else {
		logging.Infof("Found Index Defn %d", newDefnId2)
		logging.Infof("meta.Instance %v", meta.Instances)
		if meta.Instances[0].IndexerId != indexerId {
			t.Fatal(fmt.Sprintf("Index Defn %v has incorrect endpoint", newDefnId2))
		}
		if meta.Definition.WhereExpr != "TestingWhereExpr" {
			t.Fatal(fmt.Sprintf("WhereExpr is missing in Index Defn %v", newDefnId2))
		}
		if meta.Instances[0].State != common.INDEX_STATE_ACTIVE {
			t.Fatal(fmt.Sprintf("Index Defn %v has incorrect state", newDefnId2))
		}
		if meta.Instances[0].Error != "testing" {
			t.Fatal(fmt.Sprintf("Index Defn %v has incorrect error string", newDefnId2))
		}
		if meta.Instances[0].BuildTime[10] != 33 {
			t.Fatal(fmt.Sprintf("Index Defn %v has incorrect buildtime", newDefnId2))
		}
	}

	if !notifier.hasCreated {
		t.Fatal(fmt.Sprintf("Does not recieve notification for creating index %s", newDefnId))
	}
	logging.Infof(fmt.Sprintf("Recieve notification for creating index %v", newDefnId))

	if !notifier.hasDeleted {
		t.Fatal("Does not recieve notification for deleting index 101")
	}
	logging.Infof("Recieve notification for deleting index 101")

	time.Sleep(time.Duration(1000) * time.Millisecond)

	logging.Infof("Verify Cleanup / Timeout *********************************************************")

	// Create Index (immediate).

	newDefnId105, err := provider.CreateIndexWithPlan("metadata_provider_test_105", "Default", common.ForestDB,
		common.N1QL, "Testing", "TestingWhereExpr", []string{"Testing"}, false, nil)
	if err == nil {
		t.Fatal("Does not receive timeout error for create Index Defn 105 through MetadataProvider")
	}
	logging.Infof("recieve expected timeout error when creating index 105")
	close(metadata_provider_test_done)

	logging.Infof("Cleanup Test *********************************************************")

	provider.UnwatchMetadata(indexerId)
	cleanupTest(mgr, t)
	cleanSingleIndex(mgr, t, newDefnId)
	cleanSingleIndex(mgr, t, newDefnId2)
	cleanSingleIndex(mgr, t, newDefnId105)
	time.Sleep(time.Duration(1000) * time.Millisecond)
}
Example #5
0
func TestEventMgr(t *testing.T) {

	logging.SetLogLevel(logging.Trace)

	logging.Infof("Start TestEventMgr *********************************************************")

	cfg := common.SystemConfig.SectionConfig("indexer", true /*trim*/)
	cfg.Set("storage_dir", common.ConfigValue{"./data/", "metadata file path", "./"})
	os.MkdirAll("./data/", os.ModePerm)

	/*
		var requestAddr = "localhost:9885"
		var leaderAddr = "localhost:9884"
		var config = "./config.json"
	*/

	logging.Infof("Start Index Manager")
	factory := new(util.TestDefaultClientFactory)
	env := new(util.TestDefaultClientEnv)
	admin := manager.NewProjectorAdmin(factory, env, nil)
	//mgr, err := manager.NewIndexManagerInternal(requestAddr, leaderAddr, config, admin)
	mgr, err := manager.NewIndexManagerInternal("localhost:9886", "localhost:"+manager.COORD_MAINT_STREAM_PORT, admin, cfg)
	if err != nil {
		t.Fatal(err)
	}
	defer mgr.Close()

	cleanupEvtMgrTest(mgr, t)
	time.Sleep(time.Duration(1000) * time.Millisecond)

	logging.Infof("Start Listening to event")
	notifications, err := mgr.StartListenIndexCreate("TestEventMgr")
	if err != nil {
		t.Fatal(err)
	}

	// Add a new index definition : 300
	idxDefn := &common.IndexDefn{
		DefnId:          common.IndexDefnId(300),
		Name:            "event_mgr_test",
		Using:           common.ForestDB,
		Bucket:          "Default",
		IsPrimary:       false,
		SecExprs:        []string{"Testing"},
		ExprType:        common.N1QL,
		PartitionScheme: common.HASH,
		PartitionKey:    "Testing"}

	logging.Infof("Before DDL")
	err = mgr.HandleCreateIndexDDL(idxDefn)
	if err != nil {
		t.Fatal(err)
	}

	data := listen(notifications)
	if data == nil {
		t.Fatal("Does not receive notification from watcher")
	}

	idxDefn, err = common.UnmarshallIndexDefn(([]byte)(data))
	if err != nil {
		t.Fatal(err)
	}

	if idxDefn == nil {
		t.Fatal("Cannot unmarshall index definition")
	}

	if idxDefn.Name != "event_mgr_test" {
		t.Fatal("Index Definition Name mismatch")
	}

	cleanupEvtMgrTest(mgr, t)
	mgr.CleanupTopology()
	mgr.CleanupStabilityTimestamp()
	time.Sleep(time.Duration(1000) * time.Millisecond)

	logging.Infof("Stop TestEventMgr. Tearing down *********************************************************")

	mgr.Close()
	time.Sleep(time.Duration(1000) * time.Millisecond)
}