Beispiel #1
0
func TestNewRESTRouter(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	ring, err := cbgt.NewMsgRing(nil, 1)

	cfg := cbgt.NewCfgMem()
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000",
		emptyDir, "some-datasource", nil)
	r, meta, err := NewRESTRouter("v0", mgr, emptyDir, "", ring,
		AssetDir, Asset)
	if r == nil || meta == nil || err != nil {
		t.Errorf("expected no errors")
	}

	mgr = cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		[]string{"queryer", "anotherTag"},
		"", 1, "", ":1000", emptyDir, "some-datasource", nil)
	r, meta, err = NewRESTRouter("v0", mgr, emptyDir, "", ring,
		AssetDir, Asset)
	if r == nil || meta == nil || err != nil {
		t.Errorf("expected no errors")
	}
}
Beispiel #2
0
func TestManagerRestart(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	m := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil)
	if err := m.Start("wanted"); err != nil {
		t.Errorf("expected Manager.Start() to work, err: %v", err)
	}
	sourceParams := ""
	if err := m.CreateIndex("primary", "default", "123", sourceParams,
		"bleve", "foo", "", cbgt.PlanParams{},
		"bad-prevIndexUUID"); err == nil {
		t.Errorf("expected CreateIndex() err" +
			" on attempted create-with-prevIndexUUID")
	}
	if err := m.CreateIndex("primary", "default", "123", sourceParams,
		"bleve", "foo", "", cbgt.PlanParams{},
		""); err != nil {
		t.Errorf("expected CreateIndex() to work, err: %v", err)
	}
	if err := m.CreateIndex("primary", "default", "123", sourceParams,
		"bleve", "foo", "", cbgt.PlanParams{},
		"bad-prevIndexUUID"); err == nil {
		t.Errorf("expected CreateIndex() err on update" +
			" with wrong prevIndexUUID")
	}
	m.Kick("test0")
	m.PlannerNOOP("test0")
	feeds, pindexes := m.CurrentMaps()
	if len(feeds) != 1 || len(pindexes) != 1 {
		t.Errorf("expected to be 1 feed and 1 pindex,"+
			" got feeds: %+v, pindexes: %+v",
			feeds, pindexes)
	}
	for _, pindex := range pindexes {
		pindex.Dest.Close()
		if m.GetPIndex(pindex.Name) != pindex {
			t.Errorf("expected GetPIndex() to match")
		}
	}

	m2 := cbgt.NewManager(cbgt.VERSION, cfg, m.UUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil)
	if err := m2.Start("wanted"); err != nil {
		t.Errorf("expected reload Manager.Start() to work, err: %v", err)
	}
	m2.Kick("test2")
	m2.PlannerNOOP("test2")
	feeds, pindexes = m2.CurrentMaps()
	if len(feeds) != 1 || len(pindexes) != 1 {
		t.Errorf("expected to load 1 feed and 1 pindex,"+
			" got feeds: %+v, pindexes: %+v",
			feeds, pindexes)
	}
}
Beispiel #3
0
func testPartitioning(t *testing.T,
	sourceParams string,
	planParams cbgt.PlanParams,
	expectedNumPIndexes int,
	expectedNumDests int,
	andThen func(mgr *cbgt.Manager,
		sf *cbgt.PrimaryFeed, pindexes map[string]*cbgt.PIndex)) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	meh := &TestMEH{}
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
	if err := mgr.Start("wanted"); err != nil {
		t.Errorf("expected Manager.Start() to work, err: %v", err)
	}

	if err := mgr.CreateIndex("primary",
		"sourceName", "sourceUUID", sourceParams,
		"bleve", "foo", "", planParams, ""); err != nil {
		t.Errorf("expected CreateIndex() to work")
	}

	mgr.Kick("test")
	mgr.PlannerNOOP("test")
	mgr.JanitorNOOP("test")
	feeds, pindexes := mgr.CurrentMaps()
	if len(feeds) != 1 {
		t.Errorf("expected to be 1 feed, got feeds: %+v", feeds)
	}
	if len(pindexes) != expectedNumPIndexes {
		t.Errorf("expected to be %d pindex, got pindexes: %+v",
			expectedNumPIndexes, pindexes)
	}
	var feed cbgt.Feed
	for _, f := range feeds {
		feed = f
	}
	sf, ok := feed.(*cbgt.PrimaryFeed)
	if !ok || sf == nil {
		t.Errorf("expected feed to be simple")
	}
	if len(sf.Dests()) != expectedNumDests {
		t.Errorf("expected %d dests", expectedNumDests)
	}

	if andThen != nil {
		andThen(mgr, sf, pindexes)
	}
}
func (sc *ServerContext) InitCBGTManager() (base.CbgtContext, error) {

	base.LogTo("DIndex+", "Initializing CBGT")

	couchbaseUrl, err := base.CouchbaseUrlWithAuth(
		*sc.config.ClusterConfig.Server,
		sc.config.ClusterConfig.Username,
		sc.config.ClusterConfig.Password,
		*sc.config.ClusterConfig.Bucket,
	)
	if err != nil {
		return base.CbgtContext{}, err
	}

	uuid, err := cmd.MainUUID(base.IndexTypeSyncGateway, sc.config.ClusterConfig.DataDir)
	if err != nil {
		return base.CbgtContext{}, err
	}

	// this tells CBGT that we are brining a new CBGT node online
	register := "wanted"

	// use the uuid as the bindHttp so that we don't have to make the user
	// configure this, and since as far as the REST Api interface, we'll be using
	// whatever is configured in adminInterface anyway.
	// More info here:
	//   https://github.com/couchbaselabs/cbgt/issues/1
	//   https://github.com/couchbaselabs/cbgt/issues/25
	bindHttp := uuid

	options := map[string]interface{}{
		"keyPrefix": db.KSyncKeyPrefix,
	}

	cfgCb, err := cbgt.NewCfgCBEx(
		couchbaseUrl,
		*sc.config.ClusterConfig.Bucket,
		options,
	)
	if err != nil {
		return base.CbgtContext{}, err
	}

	tags := []string{"feed", "janitor", "pindex", "planner"}
	container := ""
	weight := 1                               // this would allow us to have more pindexes serviced by this node
	server := *sc.config.ClusterConfig.Server // or use "." (don't bother checking)
	extras := ""
	var managerEventHandlers cbgt.ManagerEventHandlers

	// refresh it so we have a fresh copy
	if err := cfgCb.Refresh(); err != nil {
		return base.CbgtContext{}, err
	}

	manager := cbgt.NewManager(
		cbgt.VERSION,
		cfgCb,
		uuid,
		tags,
		container,
		weight,
		extras,
		bindHttp,
		sc.config.ClusterConfig.DataDir,
		server,
		managerEventHandlers,
	)

	err = manager.Start(register)
	if err != nil {
		return base.CbgtContext{}, err
	}

	if err := sc.enableCBGTAutofailover(
		cbgt.VERSION,
		manager,
		cfgCb,
		uuid,
		couchbaseUrl,
		db.KSyncKeyPrefix,
	); err != nil {
		return base.CbgtContext{}, err
	}

	cbgtContext := base.CbgtContext{
		Manager: manager,
		Cfg:     cfgCb,
	}

	return cbgtContext, nil

}
Beispiel #5
0
func startNodeManager(testDir string, cfg cbgt.Cfg, node, register string,
	params map[string]string, server string) (
	mgr *cbgt.Manager, err error) {
	uuid := node
	if params["uuid"] != "" {
		uuid = params["uuid"]
	}
	if params[node+".uuid"] != "" {
		uuid = params[node+".uuid"]
	}

	// No planner in tags because mcp provides the planner.
	tags := []string{"feed", "pindex", "janitor", "queryer"}
	if params["tags"] != "" {
		tags = strings.Split(params["tags"], ",")
	}
	if params[node+".tags"] != "" {
		tags = strings.Split(params[node+".tags"], ",")
	}

	container := ""
	if params["container"] != "" {
		container = params["container"]
	}
	if params[node+".container"] != "" {
		container = params[node+".container"]
	}

	weight := 1
	if params["weight"] != "" {
		weight, err = strconv.Atoi(params["weight"])
	}
	if params[node+".weight"] != "" {
		weight, err = strconv.Atoi(params[node+".weight"])
	}
	if weight < 1 {
		weight = 1
	}

	extras := ""

	bindHttp := node

	dataDir := testDir + string(os.PathSeparator) + node

	os.MkdirAll(dataDir, 0700)

	meh := cbgt.ManagerEventHandlers(nil)

	mgr = cbgt.NewManager(cbgt.VERSION, cfg, uuid,
		tags, container, weight, extras,
		bindHttp, dataDir, server, meh)

	err = mgr.Start(register)
	if err != nil {
		mgr.Stop()

		return nil, err
	}

	return mgr, nil
}
Beispiel #6
0
func TestHandlersForEmptyManager(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	meh := &TestMEH{}
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
	err := mgr.Start("wanted")
	if err != nil {
		t.Errorf("expected start ok")
	}
	mgr.Kick("test-start-kick")

	mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
	mr.Write([]byte("hello"))
	mr.Write([]byte("world"))

	mgr.AddEvent([]byte(`"fizz"`))
	mgr.AddEvent([]byte(`"buzz"`))

	router, _, err := NewRESTRouter("v0", mgr, "static", "", mr,
		AssetDir, Asset)
	if err != nil || router == nil {
		t.Errorf("no mux router")
	}

	tests := []*RESTHandlerTest{
		{
			Desc:         "log on empty msg ring",
			Path:         "/api/log",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       http.StatusOK,
			ResponseBody: []byte(`{"messages":["hello","world"],"events":["fizz","buzz"]}`),
		},
		{
			Desc:   "cfg on empty manaager",
			Path:   "/api/cfg",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`"status":"ok"`:       true,
				`"indexDefs":null`:    true,
				`"nodeDefsKnown":{`:   true,
				`"nodeDefsWanted":{`:  true,
				`"planPIndexes":null`: true,
			},
		},
		{
			Desc:   "cfg refresh on empty, unchanged manager",
			Path:   "/api/cfgRefresh",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok"}`: true,
			},
		},
		{
			Desc:   "manager kick on empty, unchanged manager",
			Path:   "/api/managerKick",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok"}`: true,
			},
		},
		{
			Desc:   "manager meta",
			Path:   "/api/managerMeta",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`"status":"ok"`:    true,
				`"startSamples":{`: true,
			},
		},
		{
			Desc:   "feed stats when no feeds",
			Path:   "/api/stats",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{`: true,
				`}`: true,
			},
		},
		{
			Desc:         "list empty indexes",
			Path:         "/api/index",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       http.StatusOK,
			ResponseBody: []byte(`{"status":"ok","indexDefs":null}`),
		},
		{
			Desc:         "try to get a nonexistent index",
			Path:         "/api/index/NOT-AN-INDEX",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       400,
			ResponseBody: []byte(`index not found`),
		},
		{
			Desc:   "try to create a default index with no params",
			Path:   "/api/index/index-on-a-bad-server",
			Method: "PUT",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`rest_create_index: indexType is required`: true,
			},
		},
		{
			Desc:   "try to create a default index with no sourceType",
			Path:   "/api/index/index-on-a-bad-server",
			Method: "PUT",
			Params: url.Values{
				"indexType": []string{"no-care"},
			},
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`rest_create_index: sourceType is required`: true,
			},
		},
		{
			Desc:   "try to create a default index with bad indexType",
			Path:   "/api/index/index-on-a-bad-server",
			Method: "PUT",
			Params: url.Values{
				"indexType":  []string{"no-care"},
				"sourceType": []string{"couchbase"},
			},
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`unknown indexType`: true,
			},
		},
		{
			Desc:   "try to delete a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX",
			Method: "DELETE",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`no indexes`: true,
			},
		},
		{
			Desc:   "try to count a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX/count",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`could not get indexDefs`: true,
			},
		},
		{
			Desc:   "try to query a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX/query",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`could not get indexDefs`: true,
			},
		},
		{
			Desc:   "create an index with bogus indexType",
			Path:   "/api/index/idxBogusIndexType",
			Method: "PUT",
			Params: url.Values{
				"indexType":  []string{"not-a-real-index-type"},
				"sourceType": []string{"nil"},
			},
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`error`: true,
			},
		},
		{
			Desc:   "create a blackhole index",
			Path:   "/api/index/bh0",
			Method: "PUT",
			Params: url.Values{
				"indexType":  []string{"blackhole"},
				"sourceType": []string{"nil"},
			},
			Body:   nil,
			Status: 200,
		},
		{
			Desc:   "create a blackhole index via body",
			Path:   "/api/index/bh1",
			Method: "PUT",
			Body:   []byte(`{"type":"blackhole","sourceType":"nil"}`),
			Status: 200,
		},
		{
			Desc:   "create a blackhole index, bad params",
			Path:   "/api/index/bh2",
			Method: "PUT",
			Body:   []byte(`{"type":"blackhole","params":666,"sourceType":"nil"}`),
			Status: 400,
			ResponseMatch: map[string]bool{
				`cannot unmarshal number into Go value`: true,
			},
		},
		{
			Desc:   "create a blackhole index, bad params",
			Path:   "/api/index/bh2",
			Method: "PUT",
			Body:   []byte(`{"type":"blackhole","params":"hi","sourceType":"nil"}`),
			Status: 200,
		},
		{
			Desc:   "create a blackhole index, bad params",
			Path:   "/api/index/bh3",
			Method: "PUT",
			Body:   []byte(`{"type":"blackhole","params":{},"sourceType":"nil"}`),
			Status: 200,
		},
		{
			Desc:   "create a blackhole index, bad sourceParams",
			Path:   "/api/index/bh2s",
			Method: "PUT",
			Body:   []byte(`{"type":"blackhole","sourceParams":666,"sourceType":"nil"}`),
			Status: 400,
			ResponseMatch: map[string]bool{
				`cannot unmarshal number into Go value`: true,
			},
		},
		{
			Desc:   "create a blackhole index, bad sourceParams",
			Path:   "/api/index/bh2s",
			Method: "PUT",
			Body:   []byte(`{"type":"blackhole","sourceParams":"hi","sourceType":"nil"}`),
			Status: 200,
		},
		{
			Desc:   "create a blackhole index, bad sourceParams",
			Path:   "/api/index/bh3s",
			Method: "PUT",
			Body:   []byte(`{"type":"blackhole","sourceParams":{},"sourceType":"nil"}`),
			Status: 200,
		},
	}

	testRESTHandlers(t, tests, router)
}
Beispiel #7
0
func TestHandlersForRuntimeOps(t *testing.T) {
	emptyDir, err := ioutil.TempDir("./tmp", "test")
	if err != nil {
		t.Errorf("tempdir err: %v", err)
	}
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	meh := &TestMEH{}
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
	err = mgr.Start("wanted")
	if err != nil {
		t.Errorf("expected no start err, got: %v", err)
	}

	mgr.Kick("test-start-kick")

	mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
	mr.Write([]byte("hello"))
	mr.Write([]byte("world"))

	router, _, err := NewRESTRouter("v0", mgr, "static", "", mr,
		AssetDir, Asset)
	if err != nil || router == nil {
		t.Errorf("no mux router")
	}

	tests := []*RESTHandlerTest{
		{
			Path:   "/api/runtime",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				"arch":       true,
				"go":         true,
				"GOMAXPROCS": true,
				"GOROOT":     true,
			},
		},
		{
			Path:          "/api/runtime/args",
			Method:        "GET",
			Params:        nil,
			Body:          nil,
			Status:        http.StatusOK,
			ResponseMatch: map[string]bool{
			// Actual production args are different from "go test" context.
			},
		},
		{
			Path:         "/api/runtime/gc",
			Method:       "POST",
			Params:       nil,
			Body:         nil,
			Status:       http.StatusOK,
			ResponseBody: []byte(nil),
		},
		{
			Path:   "/api/runtime/statsMem",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				"Alloc":      true,
				"TotalAlloc": true,
			},
		},
		{
			Path:   "/api/runtime/profile/cpu",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				"incorrect or missing secs parameter": true,
			},
		},
	}

	testRESTHandlers(t, tests, router)
}
Beispiel #8
0
func TestManagerIndexControl(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	m := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil)
	if err := m.Start("wanted"); err != nil {
		t.Errorf("expected Manager.Start() to work, err: %v", err)
	}
	sourceParams := ""
	if err := m.CreateIndex("primary", "default", "123", sourceParams,
		"bleve", "foo", "", cbgt.PlanParams{}, ""); err != nil {
		t.Errorf("expected CreateIndex() to work, err: %v", err)
	}
	m.Kick("test0")
	m.PlannerNOOP("test0")

	err := m.IndexControl("foo", "wrong-uuid", "", "", "")
	if err == nil {
		t.Errorf("expected err on wrong UUID")
	}

	indexDefs, _, _ := cbgt.CfgGetIndexDefs(cfg)
	npp := indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
	if npp != nil {
		t.Errorf("expected nil npp")
	}

	err = m.IndexControl("foo", "", "", "", "")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
	if npp == nil {
		t.Errorf("expected npp")
	}
	if npp[""] != nil {
		t.Errorf("expected nil npp.sub")
	}

	err = m.IndexControl("foo", "", "disallow", "", "")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
	if npp == nil {
		t.Errorf("expected npp")
	}
	if npp[""] == nil {
		t.Errorf("expected npp.sub")
	}
	if npp[""].CanRead {
		t.Errorf("expected CanRead false")
	}
	if !npp[""].CanWrite {
		t.Errorf("expected CanWrite")
	}

	err = m.IndexControl("foo", "", "", "", "")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
	if npp == nil {
		t.Errorf("expected npp")
	}
	if npp[""] == nil {
		t.Errorf("expected npp.sub")
	}
	if npp[""].CanRead {
		t.Errorf("expected CanRead false")
	}
	if !npp[""].CanWrite {
		t.Errorf("expected CanWrite")
	}

	err = m.IndexControl("foo", "", "", "pause", "")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
	if npp == nil {
		t.Errorf("expected npp")
	}
	if npp[""] == nil {
		t.Errorf("expected npp.sub")
	}
	if npp[""].CanRead {
		t.Errorf("expected CanRead false")
	}
	if npp[""].CanWrite {
		t.Errorf("expected CanWrite false")
	}

	err = m.IndexControl("foo", "", "", "", "")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
	if npp == nil {
		t.Errorf("expected npp")
	}
	if npp[""] == nil {
		t.Errorf("expected npp.sub")
	}
	if npp[""].CanRead {
		t.Errorf("expected CanRead false")
	}
	if npp[""].CanWrite {
		t.Errorf("expected CanWrite false")
	}

	err = m.IndexControl("foo", "", "", "resume", "")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
	if npp == nil {
		t.Errorf("expected npp")
	}
	if npp[""] == nil {
		t.Errorf("expected npp.sub")
	}
	if npp[""].CanRead {
		t.Errorf("expected CanRead false")
	}
	if !npp[""].CanWrite {
		t.Errorf("expected CanWrite")
	}

	err = m.IndexControl("foo", "", "allow", "resume", "")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
	if npp == nil {
		t.Errorf("expected npp")
	}
	if npp[""] != nil {
		t.Errorf("expected nil npp.sub")
	}

	if indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen {
		t.Errorf("expected not yet frozen")
	}
	err = m.IndexControl("foo", "", "", "", "freeze")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	if !indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen {
		t.Errorf("expected frozen")
	}

	err = m.IndexControl("foo", "", "", "", "unfreeze")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	if indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen {
		t.Errorf("expected not frozen")
	}
}
Beispiel #9
0
func TestPreparePerms(t *testing.T) {

	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	meh := &TestMEH{}
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
	mgr.Start("wanted")
	mgr.Kick("test-start-kick")

	mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)

	router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
	if err != nil || router == nil {
		t.Errorf("no mux router")
	}
	router.KeepContext = true // so we can see the mux vars

	s := &stubDefinitionLookuper{
		pindexes: testPIndexesByName,
		defs: &cbgt.IndexDefs{
			IndexDefs: testIndexDefsByName,
		},
	}

	tests := []struct {
		method string
		uri    string
		body   []byte
		path   string
		perms  []string
		err    error
	}{
		// case with valid perm not containing source
		{
			method: http.MethodGet,
			uri:    "/api/index",
			path:   "/api/index",
			perms:  []string{"cluster.bucket.fts!read"},
		},
		// case with valid index name
		{
			method: http.MethodGet,
			uri:    "/api/index/i1",
			path:   "/api/index/{indexName}",
			perms:  []string{"cluster.bucket[s1].fts!read"},
		},
		// case with invalid index name
		{
			method: http.MethodGet,
			uri:    "/api/index/x1",
			path:   "/api/index/{indexName}",
			err:    errIndexNotFound,
		},
		// case with invalid index name (actuall pindex name)
		{
			method: http.MethodGet,
			uri:    "/api/index/p1",
			path:   "/api/index/{indexName}",
			err:    errIndexNotFound,
		},
		// case with valid pindex name
		{
			method: http.MethodGet,
			uri:    "/api/pindex/p1",
			path:   "/api/pindex/{pindexName}",
			perms:  []string{"cluster.bucket[s3].fts!read"},
		},
		// case with invalid pindex name
		{
			method: http.MethodGet,
			uri:    "/api/pindex/y1",
			path:   "/api/pindex/{pindexName}",
			err:    errPIndexNotFound,
		},
		// case with invalid pindex name (actually index name)
		{
			method: http.MethodGet,
			uri:    "/api/pindex/i1",
			path:   "/api/pindex/{pindexName}",
			err:    errPIndexNotFound,
		},
		// case with valid alias, but this operation does NOT expand alias
		{
			method: http.MethodGet,
			uri:    "/api/index/a1",
			path:   "/api/index/{indexName}",
			perms:  []string{"cluster.bucket.fts!read"},
		},
		// case with valid alias, and this operation DOES expand alias
		{
			method: http.MethodGet,
			uri:    "/api/index/a1/count",
			path:   "/api/index/{indexName}/count",
			perms:  []string{"cluster.bucket[s1].fts!read"},
		},
		// case with valid alias (multi), and this operation DOES expand alias
		{
			method: http.MethodGet,
			uri:    "/api/index/a2/count",
			path:   "/api/index/{indexName}/count",
			perms:  []string{"cluster.bucket[s1].fts!read", "cluster.bucket[s2].fts!read"},
		},
		// case with valid alias (containing another alias),
		// and this operation DOES expand alias
		{
			method: http.MethodGet,
			uri:    "/api/index/a3/count",
			path:   "/api/index/{indexName}/count",
			perms:  []string{"cluster.bucket[s1].fts!read", "cluster.bucket[s2].fts!read"},
		},
		// test special case for creating new index
		{
			method: http.MethodPut,
			uri:    "/api/index/anewone",
			body:   []byte(`{"type":"fulltext-index","sourceType":"couchbase","sourceName":"abucket"}`),
			path:   "/api/index/{indexName}",
			perms:  []string{"cluster.bucket[abucket].fts!write"},
		},
	}

	for i, test := range tests {
		var r io.Reader
		if test.body != nil {
			r = bytes.NewBuffer(test.body)
		}
		req, err := http.NewRequest(test.method, test.uri, r)
		if err != nil {
			t.Fatal(err)
		}

		// this actually executes things, which will usually fail
		// which is unrelated to what we're testing
		// but its the best i could do
		record := httptest.NewRecorder()
		router.ServeHTTP(record, req)

		// set the request body again, as its been consumed :(
		if test.body != nil {
			req.Body = ioutil.NopCloser(bytes.NewBuffer(test.body))
		}

		actualPerms, err := preparePerms(s, req, test.method, test.path)
		if err != test.err {
			t.Errorf("test %d, expected err %v, got %v", i, test.err, err)
		}
		sort.Strings(actualPerms)
		if !reflect.DeepEqual(actualPerms, test.perms) {
			t.Errorf("test %d, expected %v, got %v", i, test.perms, actualPerms)
		}
	}

}
Beispiel #10
0
func TestSourceNamesFromReq(t *testing.T) {

	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	meh := &TestMEH{}
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
	mgr.Start("wanted")
	mgr.Kick("test-start-kick")

	mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)

	router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
	if err != nil || router == nil {
		t.Errorf("no mux router")
	}
	router.KeepContext = true // so we can see the mux vars

	s := &stubDefinitionLookuper{
		pindexes: testPIndexesByName,
		defs: &cbgt.IndexDefs{
			IndexDefs: testIndexDefsByName,
		},
	}

	tests := []struct {
		method  string
		uri     string
		path    string
		sources []string
		err     error
	}{
		// case with valid index name
		{
			method:  http.MethodGet,
			uri:     "/api/index/i1",
			path:    "/api/index/{indexName}",
			sources: []string{"s1"},
		},
		// case with invalid index name
		{
			method: http.MethodGet,
			uri:    "/api/index/x1",
			path:   "/api/index/{indexName}",
			err:    errIndexNotFound,
		},
		// case with invalid index name (actuall pindex name)
		{
			method: http.MethodGet,
			uri:    "/api/index/p1",
			path:   "/api/index/{indexName}",
			err:    errIndexNotFound,
		},
		// case with valid pindex name
		{
			method:  http.MethodGet,
			uri:     "/api/pindex/p1",
			path:    "/api/pindex/{pindexName}",
			sources: []string{"s3"},
		},
		// case with invalid pindex name
		{
			method: http.MethodGet,
			uri:    "/api/pindex/y1",
			path:   "/api/pindex/{pindexName}",
			err:    errPIndexNotFound,
		},
		// case with invalid pindex name (actually index name)
		{
			method: http.MethodGet,
			uri:    "/api/pindex/i1",
			path:   "/api/pindex/{pindexName}",
			err:    errPIndexNotFound,
		},
		// case with valid alias, but this operation does NOT expand alias
		{
			method:  http.MethodGet,
			uri:     "/api/index/a1",
			path:    "/api/index/{indexName}",
			sources: []string{},
		},
		// case with valid alias, and this operation DOES expand alias
		{
			method:  http.MethodGet,
			uri:     "/api/index/a1/count",
			path:    "/api/index/{indexName}/count",
			sources: []string{"s1"},
		},
		// case with valid alias (multi), and this operation DOES expand alias
		{
			method:  http.MethodGet,
			uri:     "/api/index/a2/count",
			path:    "/api/index/{indexName}/count",
			sources: []string{"s1", "s2"},
		},
		// case with valid alias (containing another alias),
		// and this operation DOES expand alias
		{
			method:  http.MethodGet,
			uri:     "/api/index/a3/count",
			path:    "/api/index/{indexName}/count",
			sources: []string{"s1", "s2"},
		},
	}

	for i, test := range tests {
		req, err := http.NewRequest(test.method, test.uri, nil)
		if err != nil {
			t.Fatal(err)
		}

		// this actually executes things, which will usually fail
		// which is unrelated to what we're testing
		// but its the best i could do
		record := httptest.NewRecorder()
		router.ServeHTTP(record, req)

		actualNames, err := sourceNamesFromReq(s, req, test.method, test.path)
		if err != test.err {
			t.Errorf("test %d, expected err %v, got %v", i, test.err, err)
		}
		sort.Strings(actualNames)
		if !reflect.DeepEqual(actualNames, test.sources) {
			t.Errorf("test %d, expected %v, got %v", i, test.sources, actualNames)
		}
	}
}
func testHandlersWithOnePartitionPrimaryFeedPartialRollbackMoss(t *testing.T,
	rollbackToSeq uint64,
	rollbackDesc string,
	afterRollbackCountResponseMatch map[string]bool) {
	BlevePIndexAllowMossPrev := BlevePIndexAllowMoss
	BlevePIndexAllowMoss = true

	bleveConfigDefaultKVStorePrev := bleve.Config.DefaultKVStore
	bleve.Config.DefaultKVStore = "mossStore"

	bleveMossRegistryCollectionOptionsFTSPrev := bleveMoss.RegistryCollectionOptions["fts"]
	bleveMoss.RegistryCollectionOptions["fts"] = moss.CollectionOptions{}

	defer func() {
		BlevePIndexAllowMoss = BlevePIndexAllowMossPrev
		bleve.Config.DefaultKVStore = bleveConfigDefaultKVStorePrev
		bleveMoss.RegistryCollectionOptions["fts"] = bleveMossRegistryCollectionOptionsFTSPrev
	}()

	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	meh := &TestMEH{}
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
	mgr.Start("wanted")
	mgr.Kick("test-start-kick")

	mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)

	router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
	if err != nil || router == nil {
		t.Errorf("no mux router")
	}

	var pindex *cbgt.PIndex
	var feed *cbgt.PrimaryFeed

	waitForPersistence := func() {
		for i := 0; i < 100; i++ {
			stats := map[string]interface{}{
				"num_recs_to_persist": float64(0),
			}
			err := addPIndexStats(pindex, stats)
			if err != nil {
				t.Errorf("expected nil addPIndexStats err, got: %v", err)
			}
			v, ok := stats["num_recs_to_persist"]
			if ok {
				nrtp, ok := v.(float64)
				if ok && nrtp <= 0.0 {
					return
				}
			}
			time.Sleep(50 * time.Millisecond)
		}
		t.Errorf("persistence took too long!")
	}

	tests := []*RESTHandlerTest{
		{
			Desc:   "create dest feed with 1 partition for rollback",
			Path:   "/api/index/idx0",
			Method: "PUT",
			Params: url.Values{
				"indexType": []string{"fulltext-index"},
				"indexParams": []string{
					`{"store":{"mossStoreOptions":{"CompactionPercentage":100000.0}}}`,
				}, // Never compact during this test.
				"sourceType":   []string{"primary"},
				"sourceParams": []string{`{"numPartitions":1}`},
			},
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok"}`: true,
			},
			After: func() {
				time.Sleep(10 * time.Millisecond)
				mgr.Kick("after-rollback")
				feeds, pindexes := mgr.CurrentMaps()
				if len(feeds) != 1 {
					t.Errorf("expected to be 1 feed, got feeds: %+v", feeds)
				}
				if len(pindexes) != 1 {
					t.Errorf("expected to be 1 pindex,"+
						" got pindexes: %+v", pindexes)
				}
				for _, f := range feeds {
					var ok bool
					feed, ok = f.(*cbgt.PrimaryFeed)
					if !ok {
						t.Errorf("expected the 1 feed to be a PrimaryFeed")
					}
				}
				for _, p := range pindexes {
					pindex = p
				}
			},
		},
		{
			Before: func() {
				partition := "0"
				snapStart := uint64(1)
				snapEnd := uint64(10)
				err = feed.SnapshotStart(partition, snapStart, snapEnd)
				if err != nil {
					t.Errorf("expected no err on snapshot-start")
				}
			},
			Desc:   "count idx0 0 when snapshot just started, pre-rollback",
			Path:   "/api/index/idx0/count",
			Method: "GET",
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok","count":0}`: true,
			},
		},
		{
			Before: func() {
				partition := "0"
				key := []byte("ss1-k1")
				seq := uint64(1)
				val := []byte(`{"foo":"bar","yow":"wow"}`)
				err = feed.DataUpdate(partition, key, seq, val,
					0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
				if err != nil {
					t.Errorf("expected no err on data-update")
				}
			},
			Desc: "count idx0 should be 0 when got an update (doc created)" +
				" but snapshot not ended, pre-rollback",
			Path:   "/api/index/idx0/count",
			Method: "GET",
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok","count":0}`: true,
			},
		},
		{
			Before: func() {
				partition := "0"
				key := []byte("ss1-k2")
				seq := uint64(2)
				val := []byte(`{"foo":"bing","yow":"wow"}`)
				err = feed.DataUpdate(partition, key, seq, val,
					0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
				if err != nil {
					t.Errorf("expected no err on data-update")
				}
			},
			Desc: "count idx0 0 when got 2nd update (another doc created)" +
				" but snapshot not ended, pre-rollback",
			Path:   "/api/index/idx0/count",
			Method: "GET",
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok","count":0}`: true,
			},
		},
		{
			Before: func() {
				partition := "0"
				snapStart := uint64(11)
				snapEnd := uint64(20)
				err = feed.SnapshotStart(partition, snapStart, snapEnd)
				if err != nil {
					t.Errorf("expected no err on snapshot-start")
				}
				waitForPersistence()
			},
			Desc:   "count idx0 2 when 1st snapshot ended, pre-rollback",
			Path:   "/api/index/idx0/count",
			Method: "GET",
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok","count":2}`: true,
			},
		},
		{
			Before: func() {
				partition := "0"
				key := []byte("ss11-k11")
				seq := uint64(11)
				val := []byte(`{"foo":"baz","yow":"NOPE"}`)
				err = feed.DataUpdate(partition, key, seq, val,
					0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
				if err != nil {
					t.Errorf("expected no err on data-update")
				}
			},
			Desc: "count idx0 should be 2 when got an update (doc created)" +
				" in 2nd snapshot, 2nd snapshot not ended, pre-rollback",
			Path:   "/api/index/idx0/count",
			Method: "GET",
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok","count":2}`: true,
			},
		},
		{
			Before: func() {
				partition := "0"
				snapStart := uint64(21)
				snapEnd := uint64(30)
				err = feed.SnapshotStart(partition, snapStart, snapEnd)
				if err != nil {
					t.Errorf("expected no err on snapshot-start")
				}
				waitForPersistence()
			},
			Desc:   "count idx0 3 when 2nd snapshot ended, pre-rollback",
			Path:   "/api/index/idx0/count",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok","count":3}`: true,
			},
		},
		{
			Before: func() {
				partition := "0"
				key := []byte("ss21-k21")
				seq := uint64(21)
				val := []byte(`{"foo":"baz","yow":"whoa"}`)
				err = feed.DataUpdate(partition, key, seq, val,
					0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
				if err != nil {
					t.Errorf("expected no err on data-update")
				}
			},
			Desc: "count idx0 should be 3 when got an update (doc created)" +
				" in 3rd snapshot, 3rd snapshot not ended, pre-rollback",
			Path:   "/api/index/idx0/count",
			Method: "GET",
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok","count":3}`: true,
			},
		},
		{
			Before: func() {
				partition := "0"
				snapStart := uint64(31)
				snapEnd := uint64(40)
				err = feed.SnapshotStart(partition, snapStart, snapEnd)
				if err != nil {
					t.Errorf("expected no err on snapshot-start")
				}
				waitForPersistence()
			},
			Desc:   "count idx0 4 when 3rd snapshot ended, pre-rollback",
			Path:   "/api/index/idx0/count",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok","count":4}`: true,
			},
		},
		{
			// Rollback to a high seq number, so it's a no-op
			// w.r.t. doing an actual rollback.
			Before: func() {
				partition := "0"
				err = feed.Rollback(partition, rollbackToSeq)
				if err != nil {
					t.Errorf("expected no err on rollback, got: %v", err)
				}
				// NOTE: We might check right after rollback but before
				// we get a kick, but unfortunately results will be race-y.
				time.Sleep(100 * time.Millisecond)
				mgr.Kick("after-rollback")
			},
			Desc:          rollbackDesc,
			Path:          "/api/index/idx0/count",
			Method:        "GET",
			Params:        nil,
			Body:          nil,
			Status:        http.StatusOK,
			ResponseMatch: afterRollbackCountResponseMatch,
		},
	}

	testRESTHandlers(t, tests, router)
}
Beispiel #12
0
// Emits markdown docs of cbft's REST API.
func main() {
	rand.Seed(0)

	dataDir, _ := ioutil.TempDir("./tmp", "data")
	defer os.RemoveAll(dataDir)

	cfg := cbgt.NewCfgMem()
	tags := []string(nil)
	container := ""
	weight := 1
	extras := ""
	bindHttp := "0.0.0.0:8095"

	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		tags, container, weight, extras, bindHttp,
		dataDir, "http://localhost:8091", nil)

	sourceType := "nil"
	sourceName := ""
	sourceUUID := ""
	sourceParams := ""
	indexType := "blackhole"
	indexName := "myFirstIndex"
	indexParams := ""
	prevIndexUUID := ""

	mgr.Start("wanted")

	err := mgr.CreateIndex(
		sourceType, sourceName, sourceUUID, sourceParams,
		indexType, indexName, indexParams,
		cbgt.PlanParams{}, prevIndexUUID)
	if err != nil {
		log.Fatalf("could not create myFirstIndex, err: %v", err)
	}

	staticDir := ""
	staticETag := ""

	mr, _ := cbgt.NewMsgRing(ioutil.Discard, 1)

	router, meta, err :=
		cbft.NewRESTRouter(VERSION, mgr, staticDir, staticETag, mr)
	if err != nil {
		log.Panic(err)
	}

	mainCategoriesMap := map[string]bool{}
	mainCategories := []string(nil)

	subCategoriesMap := map[string]bool{}
	subCategories := []string(nil)

	paths := []string(nil)
	for path := range meta {
		paths = append(paths, path)

		m := meta[path]
		if m.Opts != nil {
			category := m.Opts["_category"]

			mainCategory, _, subCategory, _ := categoryParse(category)

			if !mainCategoriesMap[mainCategory] {
				mainCategoriesMap[mainCategory] = true
				mainCategories = append(mainCategories, mainCategory)
			}

			if !subCategoriesMap[subCategory] {
				subCategoriesMap[subCategory] = true
				subCategories = append(subCategories, subCategory)
			}
		}
	}

	sort.Strings(mainCategories)
	sort.Strings(subCategories)
	sort.Strings(paths)

	fmt.Printf("# API Reference\n\n")

	for _, mainCategory := range mainCategories {
		mainCategoryFirst := true

		for _, subCategory := range subCategories {
			subCategoryFirst := true

			for _, path := range paths {
				m := meta[path]

				category := ""
				if m.Opts != nil {
					category = m.Opts["_category"]
				}

				mc, mcVis, sc, scVis := categoryParse(category)
				if mc != mainCategory ||
					sc != subCategory {
					continue
				}

				if m.Opts != nil &&
					m.Opts["_status"] == "private" {
					continue
				}

				if mainCategoryFirst {
					fmt.Printf("---\n\n# %s\n\n", mcVis)
				}
				mainCategoryFirst = false

				if subCategoryFirst {
					fmt.Printf("---\n\n## %s\n\n", scVis)
				}
				subCategoryFirst = false

				pathParts := strings.Split(path, " ")
				fmt.Printf("---\n\n")
				fmt.Printf("%s `%s`\n\n", m.Method, pathParts[0])

				if m.Opts != nil && m.Opts["_about"] != "" {
					fmt.Printf("%s\n\n", m.Opts["_about"])
				}

				optNames := []string(nil)
				for optName := range m.Opts {
					if optName != "" && !strings.HasPrefix(optName, "_") {
						optNames = append(optNames, optName)
					}
				}
				sort.Strings(optNames)

				for _, optName := range optNames {
					fmt.Printf("**%s**: %s\n\n", optName, m.Opts[optName])
				}

				if m.Opts != nil && m.Opts[""] != "" {
					fmt.Printf("%s\n\n", m.Opts[""])
				}

				if m.Method == "GET" && !skipSampleResponses[pathParts[0]] {
					url := strings.Replace(pathParts[0],
						"{indexName}", "myFirstIndex", 1)

					res := sampleRequest(router, m.Method, url, nil)
					if res != nil {
						var j map[string]interface{}

						err = json.Unmarshal(res, &j)
						if err == nil {
							s, err := json.MarshalIndent(j, "    ", "  ")
							if err == nil {
								fmt.Printf("Sample response:\n\n    %s\n\n", s)
							}
						}
					}
				}
			}
		}
	}

	fmt.Printf("---\n\nCopyright (c) 2015 Couchbase, Inc.\n")
}