func TestNewRESTRouter(t *testing.T) { emptyDir, _ := ioutil.TempDir("./tmp", "test") defer os.RemoveAll(emptyDir) ring, err := cbgt.NewMsgRing(nil, 1) cfg := cbgt.NewCfgMem() mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(), nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil) r, meta, err := NewRESTRouter("v0", mgr, emptyDir, "", ring, AssetDir, Asset) if r == nil || meta == nil || err != nil { t.Errorf("expected no errors") } mgr = cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(), []string{"queryer", "anotherTag"}, "", 1, "", ":1000", emptyDir, "some-datasource", nil) r, meta, err = NewRESTRouter("v0", mgr, emptyDir, "", ring, AssetDir, Asset) if r == nil || meta == nil || err != nil { t.Errorf("expected no errors") } }
// MainUUID is a helper function for cmd-line tool developers, that // reuses a previous "baseName.uuid" file from the dataDir if it // exists, or generates a brand new UUID (and persists it). func MainUUID(baseName, dataDir string) (string, error) { uuid := cbgt.NewUUID() uuidPath := dataDir + string(os.PathSeparator) + baseName + ".uuid" uuidBuf, err := ioutil.ReadFile(uuidPath) if err == nil { uuid = strings.TrimSpace(string(uuidBuf)) if uuid == "" { return "", fmt.Errorf("error: could not parse uuidPath: %s", uuidPath) } log.Printf("main: manager uuid: %s", uuid) log.Printf("main: manager uuid was reloaded") } else { log.Printf("main: manager uuid: %s", uuid) log.Printf("main: manager uuid was generated") } err = ioutil.WriteFile(uuidPath, []byte(uuid), 0600) if err != nil { return "", fmt.Errorf("error: could not write uuidPath: %s\n"+ " Please check that your -data/-dataDir parameter (%q)\n"+ " is to a writable directory where %s can store\n"+ " index data.", uuidPath, dataDir, baseName) } return uuid, nil }
func TestMainStart(t *testing.T) { mr, err := cbgt.NewMsgRing(os.Stderr, 1000) if err != nil { t.Fatalf("unexpected error: %v", err) } router, err := MainStart(nil, cbgt.NewUUID(), nil, "", 1, "", ":1000", "bad data dir", "./static", "etag", "", "", mr, "k=v,k2=v2") if router != nil || err == nil { t.Errorf("expected empty server string to fail mainStart()") } router, err = MainStart(nil, cbgt.NewUUID(), nil, "", 1, "", ":1000", "bad data dir", "./static", "etag", "bad server", "", mr, "") if router != nil || err == nil { t.Errorf("expected bad server string to fail mainStart()") } }
func TestManagerRestart(t *testing.T) { emptyDir, _ := ioutil.TempDir("./tmp", "test") defer os.RemoveAll(emptyDir) cfg := cbgt.NewCfgMem() m := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(), nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil) if err := m.Start("wanted"); err != nil { t.Errorf("expected Manager.Start() to work, err: %v", err) } sourceParams := "" if err := m.CreateIndex("primary", "default", "123", sourceParams, "bleve", "foo", "", cbgt.PlanParams{}, "bad-prevIndexUUID"); err == nil { t.Errorf("expected CreateIndex() err" + " on attempted create-with-prevIndexUUID") } if err := m.CreateIndex("primary", "default", "123", sourceParams, "bleve", "foo", "", cbgt.PlanParams{}, ""); err != nil { t.Errorf("expected CreateIndex() to work, err: %v", err) } if err := m.CreateIndex("primary", "default", "123", sourceParams, "bleve", "foo", "", cbgt.PlanParams{}, "bad-prevIndexUUID"); err == nil { t.Errorf("expected CreateIndex() err on update" + " with wrong prevIndexUUID") } m.Kick("test0") m.PlannerNOOP("test0") feeds, pindexes := m.CurrentMaps() if len(feeds) != 1 || len(pindexes) != 1 { t.Errorf("expected to be 1 feed and 1 pindex,"+ " got feeds: %+v, pindexes: %+v", feeds, pindexes) } for _, pindex := range pindexes { pindex.Dest.Close() if m.GetPIndex(pindex.Name) != pindex { t.Errorf("expected GetPIndex() to match") } } m2 := cbgt.NewManager(cbgt.VERSION, cfg, m.UUID(), nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil) if err := m2.Start("wanted"); err != nil { t.Errorf("expected reload Manager.Start() to work, err: %v", err) } m2.Kick("test2") m2.PlannerNOOP("test2") feeds, pindexes = m2.CurrentMaps() if len(feeds) != 1 || len(pindexes) != 1 { t.Errorf("expected to load 1 feed and 1 pindex,"+ " got feeds: %+v, pindexes: %+v", feeds, pindexes) } }
func testPartitioning(t *testing.T, sourceParams string, planParams cbgt.PlanParams, expectedNumPIndexes int, expectedNumDests int, andThen func(mgr *cbgt.Manager, sf *cbgt.PrimaryFeed, pindexes map[string]*cbgt.PIndex)) { emptyDir, _ := ioutil.TempDir("./tmp", "test") defer os.RemoveAll(emptyDir) cfg := cbgt.NewCfgMem() meh := &TestMEH{} mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(), nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh) if err := mgr.Start("wanted"); err != nil { t.Errorf("expected Manager.Start() to work, err: %v", err) } if err := mgr.CreateIndex("primary", "sourceName", "sourceUUID", sourceParams, "bleve", "foo", "", planParams, ""); err != nil { t.Errorf("expected CreateIndex() to work") } mgr.Kick("test") mgr.PlannerNOOP("test") mgr.JanitorNOOP("test") feeds, pindexes := mgr.CurrentMaps() if len(feeds) != 1 { t.Errorf("expected to be 1 feed, got feeds: %+v", feeds) } if len(pindexes) != expectedNumPIndexes { t.Errorf("expected to be %d pindex, got pindexes: %+v", expectedNumPIndexes, pindexes) } var feed cbgt.Feed for _, f := range feeds { feed = f } sf, ok := feed.(*cbgt.PrimaryFeed) if !ok || sf == nil { t.Errorf("expected feed to be simple") } if len(sf.Dests()) != expectedNumDests { t.Errorf("expected %d dests", expectedNumDests) } if andThen != nil { andThen(mgr, sf, pindexes) } }
// updatePlanPIndexes_unlocked modifies the planPIndexes in/out param // based on the indexDef/node/state/op params, and may return an error // if the state transition is invalid. func (r *Rebalancer) updatePlanPIndexes_unlocked( planPIndexes *cbgt.PlanPIndexes, indexDef *cbgt.IndexDef, pindex, node, state, op string) error { planPIndex, err := r.getPlanPIndex_unlocked(planPIndexes, pindex) if err != nil { return err } canRead, canWrite := r.getNodePlanParamsReadWrite(indexDef, pindex, node) if planPIndex.Nodes == nil { planPIndex.Nodes = make(map[string]*cbgt.PlanPIndexNode) } priority := 0 if state == "replica" { priority = len(planPIndex.Nodes) } if op == "add" { if planPIndex.Nodes[node] != nil { return fmt.Errorf("updatePlanPIndexes:"+ " planPIndex already exists,"+ " indexDef: %#v, pindex: %s,"+ " node: %s, state: %q, op: %s, planPIndex: %#v", indexDef, pindex, node, state, op, planPIndex) } // TODO: Need to shift the other node priorities around? planPIndex.Nodes[node] = &cbgt.PlanPIndexNode{ CanRead: canRead, CanWrite: canWrite, Priority: priority, } } else { if planPIndex.Nodes[node] == nil { return fmt.Errorf("updatePlanPIndexes:"+ " planPIndex missing,"+ " indexDef.Name: %s, pindex: %s,"+ " node: %s, state: %q, op: %s, planPIndex: %#v", indexDef.Name, pindex, node, state, op, planPIndex) } if op == "del" { // TODO: Need to shift the other node priorities around? delete(planPIndex.Nodes, node) } else { // TODO: Need to shift the other node priorities around? planPIndex.Nodes[node] = &cbgt.PlanPIndexNode{ CanRead: canRead, CanWrite: canWrite, Priority: priority, } } } planPIndex.UUID = cbgt.NewUUID() planPIndexes.UUID = cbgt.NewUUID() planPIndexes.ImplVersion = r.version return nil }
func TestHandlersForEmptyManager(t *testing.T) { emptyDir, _ := ioutil.TempDir("./tmp", "test") defer os.RemoveAll(emptyDir) cfg := cbgt.NewCfgMem() meh := &TestMEH{} mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(), nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh) err := mgr.Start("wanted") if err != nil { t.Errorf("expected start ok") } mgr.Kick("test-start-kick") mr, _ := cbgt.NewMsgRing(os.Stderr, 1000) mr.Write([]byte("hello")) mr.Write([]byte("world")) mgr.AddEvent([]byte(`"fizz"`)) mgr.AddEvent([]byte(`"buzz"`)) router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, AssetDir, Asset) if err != nil || router == nil { t.Errorf("no mux router") } tests := []*RESTHandlerTest{ { Desc: "log on empty msg ring", Path: "/api/log", Method: "GET", Params: nil, Body: nil, Status: http.StatusOK, ResponseBody: []byte(`{"messages":["hello","world"],"events":["fizz","buzz"]}`), }, { Desc: "cfg on empty manaager", Path: "/api/cfg", Method: "GET", Params: nil, Body: nil, Status: http.StatusOK, ResponseMatch: map[string]bool{ `"status":"ok"`: true, `"indexDefs":null`: true, `"nodeDefsKnown":{`: true, `"nodeDefsWanted":{`: true, `"planPIndexes":null`: true, }, }, { Desc: "cfg refresh on empty, unchanged manager", Path: "/api/cfgRefresh", Method: "POST", Params: nil, Body: nil, Status: http.StatusOK, ResponseMatch: map[string]bool{ `{"status":"ok"}`: true, }, }, { Desc: "manager kick on empty, unchanged manager", Path: "/api/managerKick", Method: "POST", Params: nil, Body: nil, Status: http.StatusOK, ResponseMatch: map[string]bool{ `{"status":"ok"}`: true, }, }, { Desc: "manager meta", Path: "/api/managerMeta", Method: "GET", Params: nil, Body: nil, Status: http.StatusOK, ResponseMatch: map[string]bool{ `"status":"ok"`: true, `"startSamples":{`: true, }, }, { Desc: "feed stats when no feeds", Path: "/api/stats", Method: "GET", Params: nil, Body: nil, Status: http.StatusOK, ResponseMatch: map[string]bool{ `{`: true, `}`: true, }, }, { Desc: "list empty indexes", Path: "/api/index", Method: "GET", Params: nil, Body: nil, Status: http.StatusOK, ResponseBody: []byte(`{"status":"ok","indexDefs":null}`), }, { Desc: "try to get a nonexistent index", Path: "/api/index/NOT-AN-INDEX", Method: "GET", Params: nil, Body: nil, Status: 400, ResponseBody: []byte(`index not found`), }, { Desc: "try to create a default index with no params", Path: "/api/index/index-on-a-bad-server", Method: "PUT", Params: nil, Body: nil, Status: 400, ResponseMatch: map[string]bool{ `rest_create_index: indexType is required`: true, }, }, { Desc: "try to create a default index with no sourceType", Path: "/api/index/index-on-a-bad-server", Method: "PUT", Params: url.Values{ "indexType": []string{"no-care"}, }, Body: nil, Status: 400, ResponseMatch: map[string]bool{ `rest_create_index: sourceType is required`: true, }, }, { Desc: "try to create a default index with bad indexType", Path: "/api/index/index-on-a-bad-server", Method: "PUT", Params: url.Values{ "indexType": []string{"no-care"}, "sourceType": []string{"couchbase"}, }, Body: nil, Status: 400, ResponseMatch: map[string]bool{ `unknown indexType`: true, }, }, { Desc: "try to delete a nonexistent index when no indexes", Path: "/api/index/NOT-AN-INDEX", Method: "DELETE", Params: nil, Body: nil, Status: 400, ResponseMatch: map[string]bool{ `no indexes`: true, }, }, { Desc: "try to count a nonexistent index when no indexes", Path: "/api/index/NOT-AN-INDEX/count", Method: "GET", Params: nil, Body: nil, Status: 400, ResponseMatch: map[string]bool{ `could not get indexDefs`: true, }, }, { Desc: "try to query a nonexistent index when no indexes", Path: "/api/index/NOT-AN-INDEX/query", Method: "POST", Params: nil, Body: nil, Status: 400, ResponseMatch: map[string]bool{ `could not get indexDefs`: true, }, }, { Desc: "create an index with bogus indexType", Path: "/api/index/idxBogusIndexType", Method: "PUT", Params: url.Values{ "indexType": []string{"not-a-real-index-type"}, "sourceType": []string{"nil"}, }, Body: nil, Status: 400, ResponseMatch: map[string]bool{ `error`: true, }, }, { Desc: "create a blackhole index", Path: "/api/index/bh0", Method: "PUT", Params: url.Values{ "indexType": []string{"blackhole"}, "sourceType": []string{"nil"}, }, Body: nil, Status: 200, }, { Desc: "create a blackhole index via body", Path: "/api/index/bh1", Method: "PUT", Body: []byte(`{"type":"blackhole","sourceType":"nil"}`), Status: 200, }, { Desc: "create a blackhole index, bad params", Path: "/api/index/bh2", Method: "PUT", Body: []byte(`{"type":"blackhole","params":666,"sourceType":"nil"}`), Status: 400, ResponseMatch: map[string]bool{ `cannot unmarshal number into Go value`: true, }, }, { Desc: "create a blackhole index, bad params", Path: "/api/index/bh2", Method: "PUT", Body: []byte(`{"type":"blackhole","params":"hi","sourceType":"nil"}`), Status: 200, }, { Desc: "create a blackhole index, bad params", Path: "/api/index/bh3", Method: "PUT", Body: []byte(`{"type":"blackhole","params":{},"sourceType":"nil"}`), Status: 200, }, { Desc: "create a blackhole index, bad sourceParams", Path: "/api/index/bh2s", Method: "PUT", Body: []byte(`{"type":"blackhole","sourceParams":666,"sourceType":"nil"}`), Status: 400, ResponseMatch: map[string]bool{ `cannot unmarshal number into Go value`: true, }, }, { Desc: "create a blackhole index, bad sourceParams", Path: "/api/index/bh2s", Method: "PUT", Body: []byte(`{"type":"blackhole","sourceParams":"hi","sourceType":"nil"}`), Status: 200, }, { Desc: "create a blackhole index, bad sourceParams", Path: "/api/index/bh3s", Method: "PUT", Body: []byte(`{"type":"blackhole","sourceParams":{},"sourceType":"nil"}`), Status: 200, }, } testRESTHandlers(t, tests, router) }
func TestHandlersForRuntimeOps(t *testing.T) { emptyDir, err := ioutil.TempDir("./tmp", "test") if err != nil { t.Errorf("tempdir err: %v", err) } defer os.RemoveAll(emptyDir) cfg := cbgt.NewCfgMem() meh := &TestMEH{} mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(), nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh) err = mgr.Start("wanted") if err != nil { t.Errorf("expected no start err, got: %v", err) } mgr.Kick("test-start-kick") mr, _ := cbgt.NewMsgRing(os.Stderr, 1000) mr.Write([]byte("hello")) mr.Write([]byte("world")) router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, AssetDir, Asset) if err != nil || router == nil { t.Errorf("no mux router") } tests := []*RESTHandlerTest{ { Path: "/api/runtime", Method: "GET", Params: nil, Body: nil, Status: http.StatusOK, ResponseMatch: map[string]bool{ "arch": true, "go": true, "GOMAXPROCS": true, "GOROOT": true, }, }, { Path: "/api/runtime/args", Method: "GET", Params: nil, Body: nil, Status: http.StatusOK, ResponseMatch: map[string]bool{ // Actual production args are different from "go test" context. }, }, { Path: "/api/runtime/gc", Method: "POST", Params: nil, Body: nil, Status: http.StatusOK, ResponseBody: []byte(nil), }, { Path: "/api/runtime/statsMem", Method: "GET", Params: nil, Body: nil, Status: http.StatusOK, ResponseMatch: map[string]bool{ "Alloc": true, "TotalAlloc": true, }, }, { Path: "/api/runtime/profile/cpu", Method: "POST", Params: nil, Body: nil, Status: 400, ResponseMatch: map[string]bool{ "incorrect or missing secs parameter": true, }, }, } testRESTHandlers(t, tests, router) }
func TestManagerIndexControl(t *testing.T) { emptyDir, _ := ioutil.TempDir("./tmp", "test") defer os.RemoveAll(emptyDir) cfg := cbgt.NewCfgMem() m := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(), nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil) if err := m.Start("wanted"); err != nil { t.Errorf("expected Manager.Start() to work, err: %v", err) } sourceParams := "" if err := m.CreateIndex("primary", "default", "123", sourceParams, "bleve", "foo", "", cbgt.PlanParams{}, ""); err != nil { t.Errorf("expected CreateIndex() to work, err: %v", err) } m.Kick("test0") m.PlannerNOOP("test0") err := m.IndexControl("foo", "wrong-uuid", "", "", "") if err == nil { t.Errorf("expected err on wrong UUID") } indexDefs, _, _ := cbgt.CfgGetIndexDefs(cfg) npp := indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""] if npp != nil { t.Errorf("expected nil npp") } err = m.IndexControl("foo", "", "", "", "") if err != nil { t.Errorf("expected ok") } indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg) npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""] if npp == nil { t.Errorf("expected npp") } if npp[""] != nil { t.Errorf("expected nil npp.sub") } err = m.IndexControl("foo", "", "disallow", "", "") if err != nil { t.Errorf("expected ok") } indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg) npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""] if npp == nil { t.Errorf("expected npp") } if npp[""] == nil { t.Errorf("expected npp.sub") } if npp[""].CanRead { t.Errorf("expected CanRead false") } if !npp[""].CanWrite { t.Errorf("expected CanWrite") } err = m.IndexControl("foo", "", "", "", "") if err != nil { t.Errorf("expected ok") } indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg) npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""] if npp == nil { t.Errorf("expected npp") } if npp[""] == nil { t.Errorf("expected npp.sub") } if npp[""].CanRead { t.Errorf("expected CanRead false") } if !npp[""].CanWrite { t.Errorf("expected CanWrite") } err = m.IndexControl("foo", "", "", "pause", "") if err != nil { t.Errorf("expected ok") } indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg) npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""] if npp == nil { t.Errorf("expected npp") } if npp[""] == nil { t.Errorf("expected npp.sub") } if npp[""].CanRead { t.Errorf("expected CanRead false") } if npp[""].CanWrite { t.Errorf("expected CanWrite false") } err = m.IndexControl("foo", "", "", "", "") if err != nil { t.Errorf("expected ok") } indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg) npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""] if npp == nil { t.Errorf("expected npp") } if npp[""] == nil { t.Errorf("expected npp.sub") } if npp[""].CanRead { t.Errorf("expected CanRead false") } if npp[""].CanWrite { t.Errorf("expected CanWrite false") } err = m.IndexControl("foo", "", "", "resume", "") if err != nil { t.Errorf("expected ok") } indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg) npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""] if npp == nil { t.Errorf("expected npp") } if npp[""] == nil { t.Errorf("expected npp.sub") } if npp[""].CanRead { t.Errorf("expected CanRead false") } if !npp[""].CanWrite { t.Errorf("expected CanWrite") } err = m.IndexControl("foo", "", "allow", "resume", "") if err != nil { t.Errorf("expected ok") } indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg) npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""] if npp == nil { t.Errorf("expected npp") } if npp[""] != nil { t.Errorf("expected nil npp.sub") } if indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen { t.Errorf("expected not yet frozen") } err = m.IndexControl("foo", "", "", "", "freeze") if err != nil { t.Errorf("expected ok") } indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg) if !indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen { t.Errorf("expected frozen") } err = m.IndexControl("foo", "", "", "", "unfreeze") if err != nil { t.Errorf("expected ok") } indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg) if indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen { t.Errorf("expected not frozen") } }
func TestPreparePerms(t *testing.T) { emptyDir, _ := ioutil.TempDir("./tmp", "test") defer os.RemoveAll(emptyDir) cfg := cbgt.NewCfgMem() meh := &TestMEH{} mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(), nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh) mgr.Start("wanted") mgr.Kick("test-start-kick") mr, _ := cbgt.NewMsgRing(os.Stderr, 1000) router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil) if err != nil || router == nil { t.Errorf("no mux router") } router.KeepContext = true // so we can see the mux vars s := &stubDefinitionLookuper{ pindexes: testPIndexesByName, defs: &cbgt.IndexDefs{ IndexDefs: testIndexDefsByName, }, } tests := []struct { method string uri string body []byte path string perms []string err error }{ // case with valid perm not containing source { method: http.MethodGet, uri: "/api/index", path: "/api/index", perms: []string{"cluster.bucket.fts!read"}, }, // case with valid index name { method: http.MethodGet, uri: "/api/index/i1", path: "/api/index/{indexName}", perms: []string{"cluster.bucket[s1].fts!read"}, }, // case with invalid index name { method: http.MethodGet, uri: "/api/index/x1", path: "/api/index/{indexName}", err: errIndexNotFound, }, // case with invalid index name (actuall pindex name) { method: http.MethodGet, uri: "/api/index/p1", path: "/api/index/{indexName}", err: errIndexNotFound, }, // case with valid pindex name { method: http.MethodGet, uri: "/api/pindex/p1", path: "/api/pindex/{pindexName}", perms: []string{"cluster.bucket[s3].fts!read"}, }, // case with invalid pindex name { method: http.MethodGet, uri: "/api/pindex/y1", path: "/api/pindex/{pindexName}", err: errPIndexNotFound, }, // case with invalid pindex name (actually index name) { method: http.MethodGet, uri: "/api/pindex/i1", path: "/api/pindex/{pindexName}", err: errPIndexNotFound, }, // case with valid alias, but this operation does NOT expand alias { method: http.MethodGet, uri: "/api/index/a1", path: "/api/index/{indexName}", perms: []string{"cluster.bucket.fts!read"}, }, // case with valid alias, and this operation DOES expand alias { method: http.MethodGet, uri: "/api/index/a1/count", path: "/api/index/{indexName}/count", perms: []string{"cluster.bucket[s1].fts!read"}, }, // case with valid alias (multi), and this operation DOES expand alias { method: http.MethodGet, uri: "/api/index/a2/count", path: "/api/index/{indexName}/count", perms: []string{"cluster.bucket[s1].fts!read", "cluster.bucket[s2].fts!read"}, }, // case with valid alias (containing another alias), // and this operation DOES expand alias { method: http.MethodGet, uri: "/api/index/a3/count", path: "/api/index/{indexName}/count", perms: []string{"cluster.bucket[s1].fts!read", "cluster.bucket[s2].fts!read"}, }, // test special case for creating new index { method: http.MethodPut, uri: "/api/index/anewone", body: []byte(`{"type":"fulltext-index","sourceType":"couchbase","sourceName":"abucket"}`), path: "/api/index/{indexName}", perms: []string{"cluster.bucket[abucket].fts!write"}, }, } for i, test := range tests { var r io.Reader if test.body != nil { r = bytes.NewBuffer(test.body) } req, err := http.NewRequest(test.method, test.uri, r) if err != nil { t.Fatal(err) } // this actually executes things, which will usually fail // which is unrelated to what we're testing // but its the best i could do record := httptest.NewRecorder() router.ServeHTTP(record, req) // set the request body again, as its been consumed :( if test.body != nil { req.Body = ioutil.NopCloser(bytes.NewBuffer(test.body)) } actualPerms, err := preparePerms(s, req, test.method, test.path) if err != test.err { t.Errorf("test %d, expected err %v, got %v", i, test.err, err) } sort.Strings(actualPerms) if !reflect.DeepEqual(actualPerms, test.perms) { t.Errorf("test %d, expected %v, got %v", i, test.perms, actualPerms) } } }
func TestSourceNamesFromReq(t *testing.T) { emptyDir, _ := ioutil.TempDir("./tmp", "test") defer os.RemoveAll(emptyDir) cfg := cbgt.NewCfgMem() meh := &TestMEH{} mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(), nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh) mgr.Start("wanted") mgr.Kick("test-start-kick") mr, _ := cbgt.NewMsgRing(os.Stderr, 1000) router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil) if err != nil || router == nil { t.Errorf("no mux router") } router.KeepContext = true // so we can see the mux vars s := &stubDefinitionLookuper{ pindexes: testPIndexesByName, defs: &cbgt.IndexDefs{ IndexDefs: testIndexDefsByName, }, } tests := []struct { method string uri string path string sources []string err error }{ // case with valid index name { method: http.MethodGet, uri: "/api/index/i1", path: "/api/index/{indexName}", sources: []string{"s1"}, }, // case with invalid index name { method: http.MethodGet, uri: "/api/index/x1", path: "/api/index/{indexName}", err: errIndexNotFound, }, // case with invalid index name (actuall pindex name) { method: http.MethodGet, uri: "/api/index/p1", path: "/api/index/{indexName}", err: errIndexNotFound, }, // case with valid pindex name { method: http.MethodGet, uri: "/api/pindex/p1", path: "/api/pindex/{pindexName}", sources: []string{"s3"}, }, // case with invalid pindex name { method: http.MethodGet, uri: "/api/pindex/y1", path: "/api/pindex/{pindexName}", err: errPIndexNotFound, }, // case with invalid pindex name (actually index name) { method: http.MethodGet, uri: "/api/pindex/i1", path: "/api/pindex/{pindexName}", err: errPIndexNotFound, }, // case with valid alias, but this operation does NOT expand alias { method: http.MethodGet, uri: "/api/index/a1", path: "/api/index/{indexName}", sources: []string{}, }, // case with valid alias, and this operation DOES expand alias { method: http.MethodGet, uri: "/api/index/a1/count", path: "/api/index/{indexName}/count", sources: []string{"s1"}, }, // case with valid alias (multi), and this operation DOES expand alias { method: http.MethodGet, uri: "/api/index/a2/count", path: "/api/index/{indexName}/count", sources: []string{"s1", "s2"}, }, // case with valid alias (containing another alias), // and this operation DOES expand alias { method: http.MethodGet, uri: "/api/index/a3/count", path: "/api/index/{indexName}/count", sources: []string{"s1", "s2"}, }, } for i, test := range tests { req, err := http.NewRequest(test.method, test.uri, nil) if err != nil { t.Fatal(err) } // this actually executes things, which will usually fail // which is unrelated to what we're testing // but its the best i could do record := httptest.NewRecorder() router.ServeHTTP(record, req) actualNames, err := sourceNamesFromReq(s, req, test.method, test.path) if err != test.err { t.Errorf("test %d, expected err %v, got %v", i, test.err, err) } sort.Strings(actualNames) if !reflect.DeepEqual(actualNames, test.sources) { t.Errorf("test %d, expected %v, got %v", i, test.sources, actualNames) } } }
func testHandlersWithOnePartitionPrimaryFeedPartialRollbackMoss(t *testing.T, rollbackToSeq uint64, rollbackDesc string, afterRollbackCountResponseMatch map[string]bool) { BlevePIndexAllowMossPrev := BlevePIndexAllowMoss BlevePIndexAllowMoss = true bleveConfigDefaultKVStorePrev := bleve.Config.DefaultKVStore bleve.Config.DefaultKVStore = "mossStore" bleveMossRegistryCollectionOptionsFTSPrev := bleveMoss.RegistryCollectionOptions["fts"] bleveMoss.RegistryCollectionOptions["fts"] = moss.CollectionOptions{} defer func() { BlevePIndexAllowMoss = BlevePIndexAllowMossPrev bleve.Config.DefaultKVStore = bleveConfigDefaultKVStorePrev bleveMoss.RegistryCollectionOptions["fts"] = bleveMossRegistryCollectionOptionsFTSPrev }() emptyDir, _ := ioutil.TempDir("./tmp", "test") defer os.RemoveAll(emptyDir) cfg := cbgt.NewCfgMem() meh := &TestMEH{} mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(), nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh) mgr.Start("wanted") mgr.Kick("test-start-kick") mr, _ := cbgt.NewMsgRing(os.Stderr, 1000) router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil) if err != nil || router == nil { t.Errorf("no mux router") } var pindex *cbgt.PIndex var feed *cbgt.PrimaryFeed waitForPersistence := func() { for i := 0; i < 100; i++ { stats := map[string]interface{}{ "num_recs_to_persist": float64(0), } err := addPIndexStats(pindex, stats) if err != nil { t.Errorf("expected nil addPIndexStats err, got: %v", err) } v, ok := stats["num_recs_to_persist"] if ok { nrtp, ok := v.(float64) if ok && nrtp <= 0.0 { return } } time.Sleep(50 * time.Millisecond) } t.Errorf("persistence took too long!") } tests := []*RESTHandlerTest{ { Desc: "create dest feed with 1 partition for rollback", Path: "/api/index/idx0", Method: "PUT", Params: url.Values{ "indexType": []string{"fulltext-index"}, "indexParams": []string{ `{"store":{"mossStoreOptions":{"CompactionPercentage":100000.0}}}`, }, // Never compact during this test. "sourceType": []string{"primary"}, "sourceParams": []string{`{"numPartitions":1}`}, }, Status: http.StatusOK, ResponseMatch: map[string]bool{ `{"status":"ok"}`: true, }, After: func() { time.Sleep(10 * time.Millisecond) mgr.Kick("after-rollback") feeds, pindexes := mgr.CurrentMaps() if len(feeds) != 1 { t.Errorf("expected to be 1 feed, got feeds: %+v", feeds) } if len(pindexes) != 1 { t.Errorf("expected to be 1 pindex,"+ " got pindexes: %+v", pindexes) } for _, f := range feeds { var ok bool feed, ok = f.(*cbgt.PrimaryFeed) if !ok { t.Errorf("expected the 1 feed to be a PrimaryFeed") } } for _, p := range pindexes { pindex = p } }, }, { Before: func() { partition := "0" snapStart := uint64(1) snapEnd := uint64(10) err = feed.SnapshotStart(partition, snapStart, snapEnd) if err != nil { t.Errorf("expected no err on snapshot-start") } }, Desc: "count idx0 0 when snapshot just started, pre-rollback", Path: "/api/index/idx0/count", Method: "GET", Status: http.StatusOK, ResponseMatch: map[string]bool{ `{"status":"ok","count":0}`: true, }, }, { Before: func() { partition := "0" key := []byte("ss1-k1") seq := uint64(1) val := []byte(`{"foo":"bar","yow":"wow"}`) err = feed.DataUpdate(partition, key, seq, val, 0, cbgt.DEST_EXTRAS_TYPE_NIL, nil) if err != nil { t.Errorf("expected no err on data-update") } }, Desc: "count idx0 should be 0 when got an update (doc created)" + " but snapshot not ended, pre-rollback", Path: "/api/index/idx0/count", Method: "GET", Status: http.StatusOK, ResponseMatch: map[string]bool{ `{"status":"ok","count":0}`: true, }, }, { Before: func() { partition := "0" key := []byte("ss1-k2") seq := uint64(2) val := []byte(`{"foo":"bing","yow":"wow"}`) err = feed.DataUpdate(partition, key, seq, val, 0, cbgt.DEST_EXTRAS_TYPE_NIL, nil) if err != nil { t.Errorf("expected no err on data-update") } }, Desc: "count idx0 0 when got 2nd update (another doc created)" + " but snapshot not ended, pre-rollback", Path: "/api/index/idx0/count", Method: "GET", Status: http.StatusOK, ResponseMatch: map[string]bool{ `{"status":"ok","count":0}`: true, }, }, { Before: func() { partition := "0" snapStart := uint64(11) snapEnd := uint64(20) err = feed.SnapshotStart(partition, snapStart, snapEnd) if err != nil { t.Errorf("expected no err on snapshot-start") } waitForPersistence() }, Desc: "count idx0 2 when 1st snapshot ended, pre-rollback", Path: "/api/index/idx0/count", Method: "GET", Status: http.StatusOK, ResponseMatch: map[string]bool{ `{"status":"ok","count":2}`: true, }, }, { Before: func() { partition := "0" key := []byte("ss11-k11") seq := uint64(11) val := []byte(`{"foo":"baz","yow":"NOPE"}`) err = feed.DataUpdate(partition, key, seq, val, 0, cbgt.DEST_EXTRAS_TYPE_NIL, nil) if err != nil { t.Errorf("expected no err on data-update") } }, Desc: "count idx0 should be 2 when got an update (doc created)" + " in 2nd snapshot, 2nd snapshot not ended, pre-rollback", Path: "/api/index/idx0/count", Method: "GET", Status: http.StatusOK, ResponseMatch: map[string]bool{ `{"status":"ok","count":2}`: true, }, }, { Before: func() { partition := "0" snapStart := uint64(21) snapEnd := uint64(30) err = feed.SnapshotStart(partition, snapStart, snapEnd) if err != nil { t.Errorf("expected no err on snapshot-start") } waitForPersistence() }, Desc: "count idx0 3 when 2nd snapshot ended, pre-rollback", Path: "/api/index/idx0/count", Method: "GET", Params: nil, Body: nil, Status: http.StatusOK, ResponseMatch: map[string]bool{ `{"status":"ok","count":3}`: true, }, }, { Before: func() { partition := "0" key := []byte("ss21-k21") seq := uint64(21) val := []byte(`{"foo":"baz","yow":"whoa"}`) err = feed.DataUpdate(partition, key, seq, val, 0, cbgt.DEST_EXTRAS_TYPE_NIL, nil) if err != nil { t.Errorf("expected no err on data-update") } }, Desc: "count idx0 should be 3 when got an update (doc created)" + " in 3rd snapshot, 3rd snapshot not ended, pre-rollback", Path: "/api/index/idx0/count", Method: "GET", Status: http.StatusOK, ResponseMatch: map[string]bool{ `{"status":"ok","count":3}`: true, }, }, { Before: func() { partition := "0" snapStart := uint64(31) snapEnd := uint64(40) err = feed.SnapshotStart(partition, snapStart, snapEnd) if err != nil { t.Errorf("expected no err on snapshot-start") } waitForPersistence() }, Desc: "count idx0 4 when 3rd snapshot ended, pre-rollback", Path: "/api/index/idx0/count", Method: "GET", Params: nil, Body: nil, Status: http.StatusOK, ResponseMatch: map[string]bool{ `{"status":"ok","count":4}`: true, }, }, { // Rollback to a high seq number, so it's a no-op // w.r.t. doing an actual rollback. Before: func() { partition := "0" err = feed.Rollback(partition, rollbackToSeq) if err != nil { t.Errorf("expected no err on rollback, got: %v", err) } // NOTE: We might check right after rollback but before // we get a kick, but unfortunately results will be race-y. time.Sleep(100 * time.Millisecond) mgr.Kick("after-rollback") }, Desc: rollbackDesc, Path: "/api/index/idx0/count", Method: "GET", Params: nil, Body: nil, Status: http.StatusOK, ResponseMatch: afterRollbackCountResponseMatch, }, } testRESTHandlers(t, tests, router) }
// Emits markdown docs of cbft's REST API. func main() { rand.Seed(0) dataDir, _ := ioutil.TempDir("./tmp", "data") defer os.RemoveAll(dataDir) cfg := cbgt.NewCfgMem() tags := []string(nil) container := "" weight := 1 extras := "" bindHttp := "0.0.0.0:8095" mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(), tags, container, weight, extras, bindHttp, dataDir, "http://localhost:8091", nil) sourceType := "nil" sourceName := "" sourceUUID := "" sourceParams := "" indexType := "blackhole" indexName := "myFirstIndex" indexParams := "" prevIndexUUID := "" mgr.Start("wanted") err := mgr.CreateIndex( sourceType, sourceName, sourceUUID, sourceParams, indexType, indexName, indexParams, cbgt.PlanParams{}, prevIndexUUID) if err != nil { log.Fatalf("could not create myFirstIndex, err: %v", err) } staticDir := "" staticETag := "" mr, _ := cbgt.NewMsgRing(ioutil.Discard, 1) router, meta, err := cbft.NewRESTRouter(VERSION, mgr, staticDir, staticETag, mr) if err != nil { log.Panic(err) } mainCategoriesMap := map[string]bool{} mainCategories := []string(nil) subCategoriesMap := map[string]bool{} subCategories := []string(nil) paths := []string(nil) for path := range meta { paths = append(paths, path) m := meta[path] if m.Opts != nil { category := m.Opts["_category"] mainCategory, _, subCategory, _ := categoryParse(category) if !mainCategoriesMap[mainCategory] { mainCategoriesMap[mainCategory] = true mainCategories = append(mainCategories, mainCategory) } if !subCategoriesMap[subCategory] { subCategoriesMap[subCategory] = true subCategories = append(subCategories, subCategory) } } } sort.Strings(mainCategories) sort.Strings(subCategories) sort.Strings(paths) fmt.Printf("# API Reference\n\n") for _, mainCategory := range mainCategories { mainCategoryFirst := true for _, subCategory := range subCategories { subCategoryFirst := true for _, path := range paths { m := meta[path] category := "" if m.Opts != nil { category = m.Opts["_category"] } mc, mcVis, sc, scVis := categoryParse(category) if mc != mainCategory || sc != subCategory { continue } if m.Opts != nil && m.Opts["_status"] == "private" { continue } if mainCategoryFirst { fmt.Printf("---\n\n# %s\n\n", mcVis) } mainCategoryFirst = false if subCategoryFirst { fmt.Printf("---\n\n## %s\n\n", scVis) } subCategoryFirst = false pathParts := strings.Split(path, " ") fmt.Printf("---\n\n") fmt.Printf("%s `%s`\n\n", m.Method, pathParts[0]) if m.Opts != nil && m.Opts["_about"] != "" { fmt.Printf("%s\n\n", m.Opts["_about"]) } optNames := []string(nil) for optName := range m.Opts { if optName != "" && !strings.HasPrefix(optName, "_") { optNames = append(optNames, optName) } } sort.Strings(optNames) for _, optName := range optNames { fmt.Printf("**%s**: %s\n\n", optName, m.Opts[optName]) } if m.Opts != nil && m.Opts[""] != "" { fmt.Printf("%s\n\n", m.Opts[""]) } if m.Method == "GET" && !skipSampleResponses[pathParts[0]] { url := strings.Replace(pathParts[0], "{indexName}", "myFirstIndex", 1) res := sampleRequest(router, m.Method, url, nil) if res != nil { var j map[string]interface{} err = json.Unmarshal(res, &j) if err == nil { s, err := json.MarshalIndent(j, " ", " ") if err == nil { fmt.Printf("Sample response:\n\n %s\n\n", s) } } } } } } } fmt.Printf("---\n\nCopyright (c) 2015 Couchbase, Inc.\n") }