func runDeleteTest() { logging.Infof("**** Run Delete Test *******************************************") delete_test_status = make(map[uint64]*protobuf.Instance) cfg := common.SystemConfig.SectionConfig("indexer", true /*trim*/) cfg.Set("storage_dir", common.ConfigValue{"./data/", "metadata file path", "./"}) os.MkdirAll("./data/", os.ModePerm) logging.Infof("***** Start TestStreamMgr ") /* var requestAddr = "localhost:9885" var leaderAddr = "localhost:9884" */ var config = "./config.json" logging.Infof("Start Index Manager") donech := make(chan bool) factory := new(deleteTestProjectorClientFactory) factory.donech = donech env := new(deleteTestProjectorClientEnv) admin := manager.NewProjectorAdmin(factory, env, nil) //mgr, err := manager.NewIndexManagerInternal(requestAddr, leaderAddr, config, admin) mgr, err := manager.NewIndexManagerInternal("localhost:9886", "localhost:"+manager.COORD_MAINT_STREAM_PORT, admin, cfg) if err != nil { util.TT.Fatal(err) } mgr.StartCoordinator(config) time.Sleep(time.Duration(3000) * time.Millisecond) logging.Infof("Delete Test Cleanup ...") cleanupStreamMgrDeleteTest(mgr) logging.Infof("***** Run Delete Test : setup") ch := mgr.GetStabilityTimestampChannel(common.MAINT_STREAM) go runDeleteTestReceiver(ch, donech) logging.Infof("***** Run Delete Test : add 3 indexes in 2 buckets to trigger topology change") addIndexForDeleteTest(mgr) logging.Infof("***** Run Delete Test : delete index in 'Default' buckets to trigger topology change") deleteIndexForDeleteTest(mgr) logging.Infof("***** Run Delete Test : wait for stability timestamp to arrive") <-donech logging.Infof("**** Delete Test Cleanup ...") cleanupStreamMgrDeleteTest(mgr) mgr.CleanupTopology() mgr.CleanupStabilityTimestamp() time.Sleep(time.Duration(1000) * time.Millisecond) logging.Infof("**** Stop TestStreamMgr. Tearing down ") mgr.Close() time.Sleep(time.Duration(1000) * time.Millisecond) logging.Infof("**** Finish Delete Test ***********************************************************") }
func runTimerTest() { logging.Infof("**** Run Timer Test **********************************************") cfg := common.SystemConfig.SectionConfig("indexer", true /*trim*/) cfg.Set("storage_dir", common.ConfigValue{"./data/", "metadata file path", "./"}) os.MkdirAll("./data/", os.ModePerm) logging.Infof("***** Start TestStreamMgr ") /* var requestAddr = "localhost:9885" var leaderAddr = "localhost:9884" */ var config = "./config.json" logging.Infof("Start Index Manager") donech := make(chan bool) factory := new(timerTestProjectorClientFactory) factory.donech = donech env := new(timerTestProjectorClientEnv) admin := manager.NewProjectorAdmin(factory, env, nil) //mgr, err := manager.NewIndexManagerInternal(requestAddr, leaderAddr, config, admin) mgr, err := manager.NewIndexManagerInternal("localhost:9886", "localhost:"+manager.COORD_MAINT_STREAM_PORT, admin, cfg) if err != nil { util.TT.Fatal(err) } mgr.StartCoordinator(config) time.Sleep(time.Duration(3000) * time.Millisecond) mgr.SetTimestampPersistenceInterval(1) defer mgr.SetTimestampPersistenceInterval(manager.TIMESTAMP_PERSIST_INTERVAL) logging.Infof("Timer Test Cleanup ...") cleanupStreamMgrTimerTest(mgr) logging.Infof("***** Run timer Test ...") ch := mgr.GetStabilityTimestampChannel(common.MAINT_STREAM) go runTimerTestReceiver(mgr, ch, donech) logging.Infof("Setup data for Timer Test") changeTopologyForTimerTest(mgr) <-donech logging.Infof("**** Timer Test Cleanup ...") cleanupStreamMgrTimerTest(mgr) mgr.CleanupTopology() mgr.CleanupStabilityTimestamp() time.Sleep(time.Duration(1000) * time.Millisecond) logging.Infof("**** Stop TestStreamMgr. Tearing down ") mgr.Close() time.Sleep(time.Duration(1000) * time.Millisecond) logging.Infof("**** Finish Timer Test *************************************************") }
// For this test, use Index Defn Id from 100 - 110 func TestIndexManager(t *testing.T) { logging.SetLogLevel(logging.Trace) os.MkdirAll("./data/", os.ModePerm) cfg := common.SystemConfig.SectionConfig("indexer", true /*trim*/) cfg.Set("storage_dir", common.ConfigValue{"./data/", "metadata file path", "./"}) logging.Infof("Start Index Manager *********************************************************") var msgAddr = "localhost:9884" factory := new(util.TestDefaultClientFactory) env := new(util.TestDefaultClientEnv) admin := manager.NewProjectorAdmin(factory, env, nil) mgr, err := manager.NewIndexManagerInternal(msgAddr, "localhost:"+manager.COORD_MAINT_STREAM_PORT, admin, cfg) if err != nil { t.Fatal(err) } defer mgr.Close() logging.Infof("Cleanup Test *********************************************************") cleanupTest(mgr, t) logging.Infof("Setup Initial Data *********************************************************") //setupInitialData(mgr, t) logging.Infof("Start Provider *********************************************************") var providerId = "TestMetadataProvider" provider, err := client.NewMetadataProvider(providerId) if err != nil { t.Fatal(err) } defer provider.Close() provider.WatchMetadata(msgAddr) logging.Infof("Test Iterator *********************************************************") runIterator(mgr, t, 0) plan := make(map[string]interface{}) plan["nodes"] = []string{msgAddr} plan["defer_build"] = true newDefnId101, err := provider.CreateIndexWithPlan("manager_test_101", "Default", common.ForestDB, common.N1QL, "Testing", "TestingWhereExpr", []string{"Testing"}, false, plan) if err != nil { t.Fatal("Cannot create Index Defn 101 through MetadataProvider") } runIterator(mgr, t, 1) newDefnId102, err := provider.CreateIndexWithPlan("manager_test_102", "Default", common.ForestDB, common.N1QL, "Testing", "TestingWhereExpr", []string{"Testing"}, false, plan) if err != nil { t.Fatal("Cannot create Index Defn 102 through MetadataProvider") } runIterator(mgr, t, 2) logging.Infof("Cleanup Test *********************************************************") provider.UnwatchMetadata(msgAddr) cleanSingleIndex_managerTest(mgr, t, newDefnId101) cleanSingleIndex_managerTest(mgr, t, newDefnId102) time.Sleep(time.Duration(1000) * time.Millisecond) }
// For this test, use Index Defn Id from 100 - 110 func TestMetadataProvider(t *testing.T) { logging.SetLogLevel(logging.Trace) cfg := common.SystemConfig.SectionConfig("indexer", true /*trim*/) cfg.Set("storage_dir", common.ConfigValue{"./data/", "metadata file path", "./"}) os.MkdirAll("./data/", os.ModePerm) logging.Infof("Start Index Manager *********************************************************") var msgAddr = "localhost:9884" var httpAddr = "localhost:9885" factory := new(util.TestDefaultClientFactory) env := new(util.TestDefaultClientEnv) admin := manager.NewProjectorAdmin(factory, env, nil) addrPrv := util.NewFakeAddressProvider(msgAddr, httpAddr) mgr, err := manager.NewIndexManagerInternal(addrPrv, admin, cfg) if err != nil { t.Fatal(err) } defer mgr.Close() gMgr = mgr logging.Infof("Cleanup Test *********************************************************") cleanupTest(mgr, t) logging.Infof("Setup Initial Data *********************************************************") setupInitialData(mgr, t) logging.Infof("Start Provider *********************************************************") var providerId = "TestMetadataProvider" provider, err := client.NewMetadataProvider(providerId) if err != nil { t.Fatal(err) } defer provider.Close() provider.SetTimeout(int64(time.Second) * 15) indexerId, err := provider.WatchMetadata(msgAddr) if err != nil { t.Fatal(err) } // the gometa server is running in the same process as MetadataProvider (client). So sleep to // make sure that the server has a chance to finish off initialization, since the client may // be ready, but the server is not. time.Sleep(time.Duration(1000) * time.Millisecond) logging.Infof("Verify Initial Data *********************************************************") meta := lookup(provider, common.IndexDefnId(100)) if meta == nil { t.Fatal("Cannot find Index Defn 100 from MetadataProvider") } logging.Infof("found Index Defn 100") if len(meta.Instances) == 0 || meta.Instances[0].State != common.INDEX_STATE_READY { t.Fatal("Index Defn 100 state is not ready") } if meta.Instances[0].IndexerId != indexerId { t.Fatal("Index Defn 100 state is not ready") } meta = lookup(provider, common.IndexDefnId(101)) if meta == nil { t.Fatal("Cannot find Index Defn 101 from MetadataProvider") } logging.Infof("found Index Defn 101") if len(meta.Instances) == 0 || meta.Instances[0].State != common.INDEX_STATE_READY { t.Fatal("Index Defn 101 state is not ready") } if meta.Instances[0].IndexerId != indexerId { t.Fatal("Index Defn 100 state is not ready") } logging.Infof("Change Data *********************************************************") notifier := ¬ifier{hasCreated: false, hasDeleted: false} mgr.RegisterNotifier(notifier) // Create Index with deployment plan (deferred) plan := make(map[string]interface{}) plan["nodes"] = []interface{}{msgAddr} plan["defer_build"] = true newDefnId, err := provider.CreateIndexWithPlan("metadata_provider_test_102", "Default", common.ForestDB, common.N1QL, "Testing", "TestingWhereExpr", []string{"Testing"}, false, plan) if err != nil { t.Fatal("Cannot create Index Defn 102 through MetadataProvider" + err.Error()) } input := make([]common.IndexDefnId, 1) input[0] = newDefnId if err := provider.BuildIndexes(input); err != nil { t.Fatal("Cannot build Index Defn : %v", err) } logging.Infof("done creating index 102") // Drop a seeded index (created during setup step) if err := provider.DropIndex(common.IndexDefnId(101)); err != nil { t.Fatal("Cannot drop Index Defn 101 through MetadataProvider") } logging.Infof("done dropping index 101") // Create Index (immediate). newDefnId2, err := provider.CreateIndexWithPlan("metadata_provider_test_103", "Default", common.ForestDB, common.N1QL, "Testing", "TestingWhereExpr", []string{"Testing"}, false, nil) if err != nil { t.Fatal("Cannot create Index Defn 103 through MetadataProvider") } logging.Infof("done creating index 103") // Update instance (set state to ACTIVE) if err := mgr.UpdateIndexInstance("Default", newDefnId2, common.INDEX_STATE_ACTIVE, common.StreamId(100), "", nil); err != nil { t.Fatal("Fail to update index instance") } logging.Infof("done updating index 103") // Update instance (set error string) if err := mgr.UpdateIndexInstance("Default", newDefnId2, common.INDEX_STATE_NIL, common.NIL_STREAM, "testing", nil); err != nil { t.Fatal("Fail to update index instance") } logging.Infof("done updating index 103") // Create Index (immediate). This index is supposed to fail by OnIndexBuild() if _, err := provider.CreateIndexWithPlan("metadata_provider_test_104", "Default", common.ForestDB, common.N1QL, "Testing", "Testing", []string{"Testing"}, false, nil); err == nil { t.Fatal("Error does not propage for create Index Defn 104 through MetadataProvider") } logging.Infof("done creating index 104") logging.Infof("Verify Changed Data *********************************************************") if lookup(provider, common.IndexDefnId(100)) == nil { t.Fatal("Cannot find Index Defn 100 from MetadataProvider") } logging.Infof("found Index Defn 100") if lookup(provider, common.IndexDefnId(101)) != nil { t.Fatal("Found Deleted Index Defn 101 from MetadataProvider") } logging.Infof("cannot found deleted Index Defn 101") if meta = lookup(provider, newDefnId); meta == nil { t.Fatal(fmt.Sprintf("Cannot Found Index Defn %d from MetadataProvider", newDefnId)) } else { logging.Infof("Found Index Defn %d", newDefnId) logging.Infof("meta.Instance %v", meta.Instances) if meta.Instances[0].IndexerId != indexerId { t.Fatal(fmt.Sprintf("Index Defn %v has incorrect endpoint", newDefnId)) } if meta.Definition.WhereExpr != "TestingWhereExpr" { t.Fatal(fmt.Sprintf("WhereExpr is missing in Index Defn %v", newDefnId)) } if meta.Instances[0].State != common.INDEX_STATE_INITIAL { t.Fatal(fmt.Sprintf("Index Defn %v has incorrect state", newDefnId)) } } if meta = lookup(provider, newDefnId2); meta == nil { t.Fatal(fmt.Sprintf("Cannot Found Index Defn %d from MetadataProvider", newDefnId2)) } else { logging.Infof("Found Index Defn %d", newDefnId2) logging.Infof("meta.Instance %v", meta.Instances) if meta.Instances[0].IndexerId != indexerId { t.Fatal(fmt.Sprintf("Index Defn %v has incorrect endpoint", newDefnId2)) } if meta.Definition.WhereExpr != "TestingWhereExpr" { t.Fatal(fmt.Sprintf("WhereExpr is missing in Index Defn %v", newDefnId2)) } if meta.Instances[0].State != common.INDEX_STATE_ACTIVE { t.Fatal(fmt.Sprintf("Index Defn %v has incorrect state", newDefnId2)) } if meta.Instances[0].Error != "testing" { t.Fatal(fmt.Sprintf("Index Defn %v has incorrect error string", newDefnId2)) } if meta.Instances[0].BuildTime[10] != 33 { t.Fatal(fmt.Sprintf("Index Defn %v has incorrect buildtime", newDefnId2)) } } if !notifier.hasCreated { t.Fatal(fmt.Sprintf("Does not recieve notification for creating index %s", newDefnId)) } logging.Infof(fmt.Sprintf("Recieve notification for creating index %v", newDefnId)) if !notifier.hasDeleted { t.Fatal("Does not recieve notification for deleting index 101") } logging.Infof("Recieve notification for deleting index 101") time.Sleep(time.Duration(1000) * time.Millisecond) logging.Infof("Verify Cleanup / Timeout *********************************************************") // Create Index (immediate). newDefnId105, err := provider.CreateIndexWithPlan("metadata_provider_test_105", "Default", common.ForestDB, common.N1QL, "Testing", "TestingWhereExpr", []string{"Testing"}, false, nil) if err == nil { t.Fatal("Does not receive timeout error for create Index Defn 105 through MetadataProvider") } logging.Infof("recieve expected timeout error when creating index 105") close(metadata_provider_test_done) logging.Infof("Cleanup Test *********************************************************") provider.UnwatchMetadata(indexerId) cleanupTest(mgr, t) cleanSingleIndex(mgr, t, newDefnId) cleanSingleIndex(mgr, t, newDefnId2) cleanSingleIndex(mgr, t, newDefnId105) time.Sleep(time.Duration(1000) * time.Millisecond) }
func TestEventMgr(t *testing.T) { logging.SetLogLevel(logging.Trace) logging.Infof("Start TestEventMgr *********************************************************") cfg := common.SystemConfig.SectionConfig("indexer", true /*trim*/) cfg.Set("storage_dir", common.ConfigValue{"./data/", "metadata file path", "./"}) os.MkdirAll("./data/", os.ModePerm) /* var requestAddr = "localhost:9885" var leaderAddr = "localhost:9884" var config = "./config.json" */ logging.Infof("Start Index Manager") factory := new(util.TestDefaultClientFactory) env := new(util.TestDefaultClientEnv) admin := manager.NewProjectorAdmin(factory, env, nil) //mgr, err := manager.NewIndexManagerInternal(requestAddr, leaderAddr, config, admin) mgr, err := manager.NewIndexManagerInternal("localhost:9886", "localhost:"+manager.COORD_MAINT_STREAM_PORT, admin, cfg) if err != nil { t.Fatal(err) } defer mgr.Close() cleanupEvtMgrTest(mgr, t) time.Sleep(time.Duration(1000) * time.Millisecond) logging.Infof("Start Listening to event") notifications, err := mgr.StartListenIndexCreate("TestEventMgr") if err != nil { t.Fatal(err) } // Add a new index definition : 300 idxDefn := &common.IndexDefn{ DefnId: common.IndexDefnId(300), Name: "event_mgr_test", Using: common.ForestDB, Bucket: "Default", IsPrimary: false, SecExprs: []string{"Testing"}, ExprType: common.N1QL, PartitionScheme: common.HASH, PartitionKey: "Testing"} logging.Infof("Before DDL") err = mgr.HandleCreateIndexDDL(idxDefn) if err != nil { t.Fatal(err) } data := listen(notifications) if data == nil { t.Fatal("Does not receive notification from watcher") } idxDefn, err = common.UnmarshallIndexDefn(([]byte)(data)) if err != nil { t.Fatal(err) } if idxDefn == nil { t.Fatal("Cannot unmarshall index definition") } if idxDefn.Name != "event_mgr_test" { t.Fatal("Index Definition Name mismatch") } cleanupEvtMgrTest(mgr, t) mgr.CleanupTopology() mgr.CleanupStabilityTimestamp() time.Sleep(time.Duration(1000) * time.Millisecond) logging.Infof("Stop TestEventMgr. Tearing down *********************************************************") mgr.Close() time.Sleep(time.Duration(1000) * time.Millisecond) }