func TestNewRESTRouter(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
ring, err := cbgt.NewMsgRing(nil, 1)
cfg := cbgt.NewCfgMem()
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000",
emptyDir, "some-datasource", nil)
r, meta, err := NewRESTRouter("v0", mgr, emptyDir, "", ring,
AssetDir, Asset)
if r == nil || meta == nil || err != nil {
t.Errorf("expected no errors")
}
mgr = cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
[]string{"queryer", "anotherTag"},
"", 1, "", ":1000", emptyDir, "some-datasource", nil)
r, meta, err = NewRESTRouter("v0", mgr, emptyDir, "", ring,
AssetDir, Asset)
if r == nil || meta == nil || err != nil {
t.Errorf("expected no errors")
}
}
func TestManagerRestart(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
m := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil)
if err := m.Start("wanted"); err != nil {
t.Errorf("expected Manager.Start() to work, err: %v", err)
}
sourceParams := ""
if err := m.CreateIndex("primary", "default", "123", sourceParams,
"bleve", "foo", "", cbgt.PlanParams{},
"bad-prevIndexUUID"); err == nil {
t.Errorf("expected CreateIndex() err" +
" on attempted create-with-prevIndexUUID")
}
if err := m.CreateIndex("primary", "default", "123", sourceParams,
"bleve", "foo", "", cbgt.PlanParams{},
""); err != nil {
t.Errorf("expected CreateIndex() to work, err: %v", err)
}
if err := m.CreateIndex("primary", "default", "123", sourceParams,
"bleve", "foo", "", cbgt.PlanParams{},
"bad-prevIndexUUID"); err == nil {
t.Errorf("expected CreateIndex() err on update" +
" with wrong prevIndexUUID")
}
m.Kick("test0")
m.PlannerNOOP("test0")
feeds, pindexes := m.CurrentMaps()
if len(feeds) != 1 || len(pindexes) != 1 {
t.Errorf("expected to be 1 feed and 1 pindex,"+
" got feeds: %+v, pindexes: %+v",
feeds, pindexes)
}
for _, pindex := range pindexes {
pindex.Dest.Close()
if m.GetPIndex(pindex.Name) != pindex {
t.Errorf("expected GetPIndex() to match")
}
}
m2 := cbgt.NewManager(cbgt.VERSION, cfg, m.UUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil)
if err := m2.Start("wanted"); err != nil {
t.Errorf("expected reload Manager.Start() to work, err: %v", err)
}
m2.Kick("test2")
m2.PlannerNOOP("test2")
feeds, pindexes = m2.CurrentMaps()
if len(feeds) != 1 || len(pindexes) != 1 {
t.Errorf("expected to load 1 feed and 1 pindex,"+
" got feeds: %+v, pindexes: %+v",
feeds, pindexes)
}
}
func testPartitioning(t *testing.T,
sourceParams string,
planParams cbgt.PlanParams,
expectedNumPIndexes int,
expectedNumDests int,
andThen func(mgr *cbgt.Manager,
sf *cbgt.PrimaryFeed, pindexes map[string]*cbgt.PIndex)) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
if err := mgr.Start("wanted"); err != nil {
t.Errorf("expected Manager.Start() to work, err: %v", err)
}
if err := mgr.CreateIndex("primary",
"sourceName", "sourceUUID", sourceParams,
"bleve", "foo", "", planParams, ""); err != nil {
t.Errorf("expected CreateIndex() to work")
}
mgr.Kick("test")
mgr.PlannerNOOP("test")
mgr.JanitorNOOP("test")
feeds, pindexes := mgr.CurrentMaps()
if len(feeds) != 1 {
t.Errorf("expected to be 1 feed, got feeds: %+v", feeds)
}
if len(pindexes) != expectedNumPIndexes {
t.Errorf("expected to be %d pindex, got pindexes: %+v",
expectedNumPIndexes, pindexes)
}
var feed cbgt.Feed
for _, f := range feeds {
feed = f
}
sf, ok := feed.(*cbgt.PrimaryFeed)
if !ok || sf == nil {
t.Errorf("expected feed to be simple")
}
if len(sf.Dests()) != expectedNumDests {
t.Errorf("expected %d dests", expectedNumDests)
}
if andThen != nil {
andThen(mgr, sf, pindexes)
}
}
func (sc *ServerContext) InitCBGTManager() (base.CbgtContext, error) {
base.LogTo("DIndex+", "Initializing CBGT")
couchbaseUrl, err := base.CouchbaseUrlWithAuth(
*sc.config.ClusterConfig.Server,
sc.config.ClusterConfig.Username,
sc.config.ClusterConfig.Password,
*sc.config.ClusterConfig.Bucket,
)
if err != nil {
return base.CbgtContext{}, err
}
uuid, err := cmd.MainUUID(base.IndexTypeSyncGateway, sc.config.ClusterConfig.DataDir)
if err != nil {
return base.CbgtContext{}, err
}
// this tells CBGT that we are brining a new CBGT node online
register := "wanted"
// use the uuid as the bindHttp so that we don't have to make the user
// configure this, and since as far as the REST Api interface, we'll be using
// whatever is configured in adminInterface anyway.
// More info here:
// https://github.com/couchbaselabs/cbgt/issues/1
// https://github.com/couchbaselabs/cbgt/issues/25
bindHttp := uuid
options := map[string]interface{}{
"keyPrefix": db.KSyncKeyPrefix,
}
cfgCb, err := cbgt.NewCfgCBEx(
couchbaseUrl,
*sc.config.ClusterConfig.Bucket,
options,
)
if err != nil {
return base.CbgtContext{}, err
}
tags := []string{"feed", "janitor", "pindex", "planner"}
container := ""
weight := 1 // this would allow us to have more pindexes serviced by this node
server := *sc.config.ClusterConfig.Server // or use "." (don't bother checking)
extras := ""
var managerEventHandlers cbgt.ManagerEventHandlers
// refresh it so we have a fresh copy
if err := cfgCb.Refresh(); err != nil {
return base.CbgtContext{}, err
}
manager := cbgt.NewManager(
cbgt.VERSION,
cfgCb,
uuid,
tags,
container,
weight,
extras,
bindHttp,
sc.config.ClusterConfig.DataDir,
server,
managerEventHandlers,
)
err = manager.Start(register)
if err != nil {
return base.CbgtContext{}, err
}
if err := sc.enableCBGTAutofailover(
cbgt.VERSION,
manager,
cfgCb,
uuid,
couchbaseUrl,
db.KSyncKeyPrefix,
); err != nil {
return base.CbgtContext{}, err
}
cbgtContext := base.CbgtContext{
Manager: manager,
Cfg: cfgCb,
}
return cbgtContext, nil
}
func startNodeManager(testDir string, cfg cbgt.Cfg, node, register string,
params map[string]string, server string) (
mgr *cbgt.Manager, err error) {
uuid := node
if params["uuid"] != "" {
uuid = params["uuid"]
}
if params[node+".uuid"] != "" {
uuid = params[node+".uuid"]
}
// No planner in tags because mcp provides the planner.
tags := []string{"feed", "pindex", "janitor", "queryer"}
if params["tags"] != "" {
tags = strings.Split(params["tags"], ",")
}
if params[node+".tags"] != "" {
tags = strings.Split(params[node+".tags"], ",")
}
container := ""
if params["container"] != "" {
container = params["container"]
}
if params[node+".container"] != "" {
container = params[node+".container"]
}
weight := 1
if params["weight"] != "" {
weight, err = strconv.Atoi(params["weight"])
}
if params[node+".weight"] != "" {
weight, err = strconv.Atoi(params[node+".weight"])
}
if weight < 1 {
weight = 1
}
extras := ""
bindHttp := node
dataDir := testDir + string(os.PathSeparator) + node
os.MkdirAll(dataDir, 0700)
meh := cbgt.ManagerEventHandlers(nil)
mgr = cbgt.NewManager(cbgt.VERSION, cfg, uuid,
tags, container, weight, extras,
bindHttp, dataDir, server, meh)
err = mgr.Start(register)
if err != nil {
mgr.Stop()
return nil, err
}
return mgr, nil
}
func TestHandlersForEmptyManager(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
err := mgr.Start("wanted")
if err != nil {
t.Errorf("expected start ok")
}
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
mr.Write([]byte("hello"))
mr.Write([]byte("world"))
mgr.AddEvent([]byte(`"fizz"`))
mgr.AddEvent([]byte(`"buzz"`))
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr,
AssetDir, Asset)
if err != nil || router == nil {
t.Errorf("no mux router")
}
tests := []*RESTHandlerTest{
{
Desc: "log on empty msg ring",
Path: "/api/log",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseBody: []byte(`{"messages":["hello","world"],"events":["fizz","buzz"]}`),
},
{
Desc: "cfg on empty manaager",
Path: "/api/cfg",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`"status":"ok"`: true,
`"indexDefs":null`: true,
`"nodeDefsKnown":{`: true,
`"nodeDefsWanted":{`: true,
`"planPIndexes":null`: true,
},
},
{
Desc: "cfg refresh on empty, unchanged manager",
Path: "/api/cfgRefresh",
Method: "POST",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok"}`: true,
},
},
{
Desc: "manager kick on empty, unchanged manager",
Path: "/api/managerKick",
Method: "POST",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok"}`: true,
},
},
{
Desc: "manager meta",
Path: "/api/managerMeta",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`"status":"ok"`: true,
`"startSamples":{`: true,
},
},
{
Desc: "feed stats when no feeds",
Path: "/api/stats",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{`: true,
`}`: true,
},
},
{
Desc: "list empty indexes",
Path: "/api/index",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseBody: []byte(`{"status":"ok","indexDefs":null}`),
},
{
Desc: "try to get a nonexistent index",
Path: "/api/index/NOT-AN-INDEX",
Method: "GET",
Params: nil,
Body: nil,
Status: 400,
ResponseBody: []byte(`index not found`),
},
{
Desc: "try to create a default index with no params",
Path: "/api/index/index-on-a-bad-server",
Method: "PUT",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`rest_create_index: indexType is required`: true,
},
},
{
Desc: "try to create a default index with no sourceType",
Path: "/api/index/index-on-a-bad-server",
Method: "PUT",
Params: url.Values{
"indexType": []string{"no-care"},
},
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`rest_create_index: sourceType is required`: true,
},
},
{
Desc: "try to create a default index with bad indexType",
Path: "/api/index/index-on-a-bad-server",
Method: "PUT",
Params: url.Values{
"indexType": []string{"no-care"},
"sourceType": []string{"couchbase"},
},
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`unknown indexType`: true,
},
},
{
Desc: "try to delete a nonexistent index when no indexes",
Path: "/api/index/NOT-AN-INDEX",
Method: "DELETE",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`no indexes`: true,
},
},
{
Desc: "try to count a nonexistent index when no indexes",
Path: "/api/index/NOT-AN-INDEX/count",
Method: "GET",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`could not get indexDefs`: true,
},
},
{
Desc: "try to query a nonexistent index when no indexes",
Path: "/api/index/NOT-AN-INDEX/query",
Method: "POST",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`could not get indexDefs`: true,
},
},
{
Desc: "create an index with bogus indexType",
Path: "/api/index/idxBogusIndexType",
Method: "PUT",
Params: url.Values{
"indexType": []string{"not-a-real-index-type"},
"sourceType": []string{"nil"},
},
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`error`: true,
},
},
{
Desc: "create a blackhole index",
Path: "/api/index/bh0",
Method: "PUT",
Params: url.Values{
"indexType": []string{"blackhole"},
"sourceType": []string{"nil"},
},
Body: nil,
Status: 200,
},
{
Desc: "create a blackhole index via body",
Path: "/api/index/bh1",
Method: "PUT",
Body: []byte(`{"type":"blackhole","sourceType":"nil"}`),
Status: 200,
},
{
Desc: "create a blackhole index, bad params",
Path: "/api/index/bh2",
Method: "PUT",
Body: []byte(`{"type":"blackhole","params":666,"sourceType":"nil"}`),
Status: 400,
ResponseMatch: map[string]bool{
`cannot unmarshal number into Go value`: true,
},
},
{
Desc: "create a blackhole index, bad params",
Path: "/api/index/bh2",
Method: "PUT",
Body: []byte(`{"type":"blackhole","params":"hi","sourceType":"nil"}`),
Status: 200,
},
{
Desc: "create a blackhole index, bad params",
Path: "/api/index/bh3",
Method: "PUT",
Body: []byte(`{"type":"blackhole","params":{},"sourceType":"nil"}`),
Status: 200,
},
{
Desc: "create a blackhole index, bad sourceParams",
Path: "/api/index/bh2s",
Method: "PUT",
Body: []byte(`{"type":"blackhole","sourceParams":666,"sourceType":"nil"}`),
Status: 400,
ResponseMatch: map[string]bool{
`cannot unmarshal number into Go value`: true,
},
},
{
Desc: "create a blackhole index, bad sourceParams",
Path: "/api/index/bh2s",
Method: "PUT",
Body: []byte(`{"type":"blackhole","sourceParams":"hi","sourceType":"nil"}`),
Status: 200,
},
{
Desc: "create a blackhole index, bad sourceParams",
Path: "/api/index/bh3s",
Method: "PUT",
Body: []byte(`{"type":"blackhole","sourceParams":{},"sourceType":"nil"}`),
Status: 200,
},
}
testRESTHandlers(t, tests, router)
}
func TestHandlersForRuntimeOps(t *testing.T) {
emptyDir, err := ioutil.TempDir("./tmp", "test")
if err != nil {
t.Errorf("tempdir err: %v", err)
}
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
err = mgr.Start("wanted")
if err != nil {
t.Errorf("expected no start err, got: %v", err)
}
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
mr.Write([]byte("hello"))
mr.Write([]byte("world"))
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr,
AssetDir, Asset)
if err != nil || router == nil {
t.Errorf("no mux router")
}
tests := []*RESTHandlerTest{
{
Path: "/api/runtime",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
"arch": true,
"go": true,
"GOMAXPROCS": true,
"GOROOT": true,
},
},
{
Path: "/api/runtime/args",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
// Actual production args are different from "go test" context.
},
},
{
Path: "/api/runtime/gc",
Method: "POST",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseBody: []byte(nil),
},
{
Path: "/api/runtime/statsMem",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
"Alloc": true,
"TotalAlloc": true,
},
},
{
Path: "/api/runtime/profile/cpu",
Method: "POST",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
"incorrect or missing secs parameter": true,
},
},
}
testRESTHandlers(t, tests, router)
}
func TestManagerIndexControl(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
m := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil)
if err := m.Start("wanted"); err != nil {
t.Errorf("expected Manager.Start() to work, err: %v", err)
}
sourceParams := ""
if err := m.CreateIndex("primary", "default", "123", sourceParams,
"bleve", "foo", "", cbgt.PlanParams{}, ""); err != nil {
t.Errorf("expected CreateIndex() to work, err: %v", err)
}
m.Kick("test0")
m.PlannerNOOP("test0")
err := m.IndexControl("foo", "wrong-uuid", "", "", "")
if err == nil {
t.Errorf("expected err on wrong UUID")
}
indexDefs, _, _ := cbgt.CfgGetIndexDefs(cfg)
npp := indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp != nil {
t.Errorf("expected nil npp")
}
err = m.IndexControl("foo", "", "", "", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] != nil {
t.Errorf("expected nil npp.sub")
}
err = m.IndexControl("foo", "", "disallow", "", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] == nil {
t.Errorf("expected npp.sub")
}
if npp[""].CanRead {
t.Errorf("expected CanRead false")
}
if !npp[""].CanWrite {
t.Errorf("expected CanWrite")
}
err = m.IndexControl("foo", "", "", "", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] == nil {
t.Errorf("expected npp.sub")
}
if npp[""].CanRead {
t.Errorf("expected CanRead false")
}
if !npp[""].CanWrite {
t.Errorf("expected CanWrite")
}
err = m.IndexControl("foo", "", "", "pause", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] == nil {
t.Errorf("expected npp.sub")
}
if npp[""].CanRead {
t.Errorf("expected CanRead false")
}
if npp[""].CanWrite {
t.Errorf("expected CanWrite false")
}
err = m.IndexControl("foo", "", "", "", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] == nil {
t.Errorf("expected npp.sub")
}
if npp[""].CanRead {
t.Errorf("expected CanRead false")
}
if npp[""].CanWrite {
t.Errorf("expected CanWrite false")
}
err = m.IndexControl("foo", "", "", "resume", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] == nil {
t.Errorf("expected npp.sub")
}
if npp[""].CanRead {
t.Errorf("expected CanRead false")
}
if !npp[""].CanWrite {
t.Errorf("expected CanWrite")
}
err = m.IndexControl("foo", "", "allow", "resume", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] != nil {
t.Errorf("expected nil npp.sub")
}
if indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen {
t.Errorf("expected not yet frozen")
}
err = m.IndexControl("foo", "", "", "", "freeze")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
if !indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen {
t.Errorf("expected frozen")
}
err = m.IndexControl("foo", "", "", "", "unfreeze")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
if indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen {
t.Errorf("expected not frozen")
}
}
func TestPreparePerms(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
mgr.Start("wanted")
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
if err != nil || router == nil {
t.Errorf("no mux router")
}
router.KeepContext = true // so we can see the mux vars
s := &stubDefinitionLookuper{
pindexes: testPIndexesByName,
defs: &cbgt.IndexDefs{
IndexDefs: testIndexDefsByName,
},
}
tests := []struct {
method string
uri string
body []byte
path string
perms []string
err error
}{
// case with valid perm not containing source
{
method: http.MethodGet,
uri: "/api/index",
path: "/api/index",
perms: []string{"cluster.bucket.fts!read"},
},
// case with valid index name
{
method: http.MethodGet,
uri: "/api/index/i1",
path: "/api/index/{indexName}",
perms: []string{"cluster.bucket[s1].fts!read"},
},
// case with invalid index name
{
method: http.MethodGet,
uri: "/api/index/x1",
path: "/api/index/{indexName}",
err: errIndexNotFound,
},
// case with invalid index name (actuall pindex name)
{
method: http.MethodGet,
uri: "/api/index/p1",
path: "/api/index/{indexName}",
err: errIndexNotFound,
},
// case with valid pindex name
{
method: http.MethodGet,
uri: "/api/pindex/p1",
path: "/api/pindex/{pindexName}",
perms: []string{"cluster.bucket[s3].fts!read"},
},
// case with invalid pindex name
{
method: http.MethodGet,
uri: "/api/pindex/y1",
path: "/api/pindex/{pindexName}",
err: errPIndexNotFound,
},
// case with invalid pindex name (actually index name)
{
method: http.MethodGet,
uri: "/api/pindex/i1",
path: "/api/pindex/{pindexName}",
err: errPIndexNotFound,
},
// case with valid alias, but this operation does NOT expand alias
{
method: http.MethodGet,
uri: "/api/index/a1",
path: "/api/index/{indexName}",
perms: []string{"cluster.bucket.fts!read"},
},
// case with valid alias, and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a1/count",
path: "/api/index/{indexName}/count",
perms: []string{"cluster.bucket[s1].fts!read"},
},
// case with valid alias (multi), and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a2/count",
path: "/api/index/{indexName}/count",
perms: []string{"cluster.bucket[s1].fts!read", "cluster.bucket[s2].fts!read"},
},
// case with valid alias (containing another alias),
// and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a3/count",
path: "/api/index/{indexName}/count",
perms: []string{"cluster.bucket[s1].fts!read", "cluster.bucket[s2].fts!read"},
},
// test special case for creating new index
{
method: http.MethodPut,
uri: "/api/index/anewone",
body: []byte(`{"type":"fulltext-index","sourceType":"couchbase","sourceName":"abucket"}`),
path: "/api/index/{indexName}",
perms: []string{"cluster.bucket[abucket].fts!write"},
},
}
for i, test := range tests {
var r io.Reader
if test.body != nil {
r = bytes.NewBuffer(test.body)
}
req, err := http.NewRequest(test.method, test.uri, r)
if err != nil {
t.Fatal(err)
}
// this actually executes things, which will usually fail
// which is unrelated to what we're testing
// but its the best i could do
record := httptest.NewRecorder()
router.ServeHTTP(record, req)
// set the request body again, as its been consumed :(
if test.body != nil {
req.Body = ioutil.NopCloser(bytes.NewBuffer(test.body))
}
actualPerms, err := preparePerms(s, req, test.method, test.path)
if err != test.err {
t.Errorf("test %d, expected err %v, got %v", i, test.err, err)
}
sort.Strings(actualPerms)
if !reflect.DeepEqual(actualPerms, test.perms) {
t.Errorf("test %d, expected %v, got %v", i, test.perms, actualPerms)
}
}
}
func TestSourceNamesFromReq(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
mgr.Start("wanted")
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
if err != nil || router == nil {
t.Errorf("no mux router")
}
router.KeepContext = true // so we can see the mux vars
s := &stubDefinitionLookuper{
pindexes: testPIndexesByName,
defs: &cbgt.IndexDefs{
IndexDefs: testIndexDefsByName,
},
}
tests := []struct {
method string
uri string
path string
sources []string
err error
}{
// case with valid index name
{
method: http.MethodGet,
uri: "/api/index/i1",
path: "/api/index/{indexName}",
sources: []string{"s1"},
},
// case with invalid index name
{
method: http.MethodGet,
uri: "/api/index/x1",
path: "/api/index/{indexName}",
err: errIndexNotFound,
},
// case with invalid index name (actuall pindex name)
{
method: http.MethodGet,
uri: "/api/index/p1",
path: "/api/index/{indexName}",
err: errIndexNotFound,
},
// case with valid pindex name
{
method: http.MethodGet,
uri: "/api/pindex/p1",
path: "/api/pindex/{pindexName}",
sources: []string{"s3"},
},
// case with invalid pindex name
{
method: http.MethodGet,
uri: "/api/pindex/y1",
path: "/api/pindex/{pindexName}",
err: errPIndexNotFound,
},
// case with invalid pindex name (actually index name)
{
method: http.MethodGet,
uri: "/api/pindex/i1",
path: "/api/pindex/{pindexName}",
err: errPIndexNotFound,
},
// case with valid alias, but this operation does NOT expand alias
{
method: http.MethodGet,
uri: "/api/index/a1",
path: "/api/index/{indexName}",
sources: []string{},
},
// case with valid alias, and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a1/count",
path: "/api/index/{indexName}/count",
sources: []string{"s1"},
},
// case with valid alias (multi), and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a2/count",
path: "/api/index/{indexName}/count",
sources: []string{"s1", "s2"},
},
// case with valid alias (containing another alias),
// and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a3/count",
path: "/api/index/{indexName}/count",
sources: []string{"s1", "s2"},
},
}
for i, test := range tests {
req, err := http.NewRequest(test.method, test.uri, nil)
if err != nil {
t.Fatal(err)
}
// this actually executes things, which will usually fail
// which is unrelated to what we're testing
// but its the best i could do
record := httptest.NewRecorder()
router.ServeHTTP(record, req)
actualNames, err := sourceNamesFromReq(s, req, test.method, test.path)
if err != test.err {
t.Errorf("test %d, expected err %v, got %v", i, test.err, err)
}
sort.Strings(actualNames)
if !reflect.DeepEqual(actualNames, test.sources) {
t.Errorf("test %d, expected %v, got %v", i, test.sources, actualNames)
}
}
}
func testHandlersWithOnePartitionPrimaryFeedPartialRollbackMoss(t *testing.T,
rollbackToSeq uint64,
rollbackDesc string,
afterRollbackCountResponseMatch map[string]bool) {
BlevePIndexAllowMossPrev := BlevePIndexAllowMoss
BlevePIndexAllowMoss = true
bleveConfigDefaultKVStorePrev := bleve.Config.DefaultKVStore
bleve.Config.DefaultKVStore = "mossStore"
bleveMossRegistryCollectionOptionsFTSPrev := bleveMoss.RegistryCollectionOptions["fts"]
bleveMoss.RegistryCollectionOptions["fts"] = moss.CollectionOptions{}
defer func() {
BlevePIndexAllowMoss = BlevePIndexAllowMossPrev
bleve.Config.DefaultKVStore = bleveConfigDefaultKVStorePrev
bleveMoss.RegistryCollectionOptions["fts"] = bleveMossRegistryCollectionOptionsFTSPrev
}()
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
mgr.Start("wanted")
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
if err != nil || router == nil {
t.Errorf("no mux router")
}
var pindex *cbgt.PIndex
var feed *cbgt.PrimaryFeed
waitForPersistence := func() {
for i := 0; i < 100; i++ {
stats := map[string]interface{}{
"num_recs_to_persist": float64(0),
}
err := addPIndexStats(pindex, stats)
if err != nil {
t.Errorf("expected nil addPIndexStats err, got: %v", err)
}
v, ok := stats["num_recs_to_persist"]
if ok {
nrtp, ok := v.(float64)
if ok && nrtp <= 0.0 {
return
}
}
time.Sleep(50 * time.Millisecond)
}
t.Errorf("persistence took too long!")
}
tests := []*RESTHandlerTest{
{
Desc: "create dest feed with 1 partition for rollback",
Path: "/api/index/idx0",
Method: "PUT",
Params: url.Values{
"indexType": []string{"fulltext-index"},
"indexParams": []string{
`{"store":{"mossStoreOptions":{"CompactionPercentage":100000.0}}}`,
}, // Never compact during this test.
"sourceType": []string{"primary"},
"sourceParams": []string{`{"numPartitions":1}`},
},
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok"}`: true,
},
After: func() {
time.Sleep(10 * time.Millisecond)
mgr.Kick("after-rollback")
feeds, pindexes := mgr.CurrentMaps()
if len(feeds) != 1 {
t.Errorf("expected to be 1 feed, got feeds: %+v", feeds)
}
if len(pindexes) != 1 {
t.Errorf("expected to be 1 pindex,"+
" got pindexes: %+v", pindexes)
}
for _, f := range feeds {
var ok bool
feed, ok = f.(*cbgt.PrimaryFeed)
if !ok {
t.Errorf("expected the 1 feed to be a PrimaryFeed")
}
}
for _, p := range pindexes {
pindex = p
}
},
},
{
Before: func() {
partition := "0"
snapStart := uint64(1)
snapEnd := uint64(10)
err = feed.SnapshotStart(partition, snapStart, snapEnd)
if err != nil {
t.Errorf("expected no err on snapshot-start")
}
},
Desc: "count idx0 0 when snapshot just started, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":0}`: true,
},
},
{
Before: func() {
partition := "0"
key := []byte("ss1-k1")
seq := uint64(1)
val := []byte(`{"foo":"bar","yow":"wow"}`)
err = feed.DataUpdate(partition, key, seq, val,
0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
if err != nil {
t.Errorf("expected no err on data-update")
}
},
Desc: "count idx0 should be 0 when got an update (doc created)" +
" but snapshot not ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":0}`: true,
},
},
{
Before: func() {
partition := "0"
key := []byte("ss1-k2")
seq := uint64(2)
val := []byte(`{"foo":"bing","yow":"wow"}`)
err = feed.DataUpdate(partition, key, seq, val,
0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
if err != nil {
t.Errorf("expected no err on data-update")
}
},
Desc: "count idx0 0 when got 2nd update (another doc created)" +
" but snapshot not ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":0}`: true,
},
},
{
Before: func() {
partition := "0"
snapStart := uint64(11)
snapEnd := uint64(20)
err = feed.SnapshotStart(partition, snapStart, snapEnd)
if err != nil {
t.Errorf("expected no err on snapshot-start")
}
waitForPersistence()
},
Desc: "count idx0 2 when 1st snapshot ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":2}`: true,
},
},
{
Before: func() {
partition := "0"
key := []byte("ss11-k11")
seq := uint64(11)
val := []byte(`{"foo":"baz","yow":"NOPE"}`)
err = feed.DataUpdate(partition, key, seq, val,
0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
if err != nil {
t.Errorf("expected no err on data-update")
}
},
Desc: "count idx0 should be 2 when got an update (doc created)" +
" in 2nd snapshot, 2nd snapshot not ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":2}`: true,
},
},
{
Before: func() {
partition := "0"
snapStart := uint64(21)
snapEnd := uint64(30)
err = feed.SnapshotStart(partition, snapStart, snapEnd)
if err != nil {
t.Errorf("expected no err on snapshot-start")
}
waitForPersistence()
},
Desc: "count idx0 3 when 2nd snapshot ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":3}`: true,
},
},
{
Before: func() {
partition := "0"
key := []byte("ss21-k21")
seq := uint64(21)
val := []byte(`{"foo":"baz","yow":"whoa"}`)
err = feed.DataUpdate(partition, key, seq, val,
0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
if err != nil {
t.Errorf("expected no err on data-update")
}
},
Desc: "count idx0 should be 3 when got an update (doc created)" +
" in 3rd snapshot, 3rd snapshot not ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":3}`: true,
},
},
{
Before: func() {
partition := "0"
snapStart := uint64(31)
snapEnd := uint64(40)
err = feed.SnapshotStart(partition, snapStart, snapEnd)
if err != nil {
t.Errorf("expected no err on snapshot-start")
}
waitForPersistence()
},
Desc: "count idx0 4 when 3rd snapshot ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":4}`: true,
},
},
{
// Rollback to a high seq number, so it's a no-op
// w.r.t. doing an actual rollback.
Before: func() {
partition := "0"
err = feed.Rollback(partition, rollbackToSeq)
if err != nil {
t.Errorf("expected no err on rollback, got: %v", err)
}
// NOTE: We might check right after rollback but before
// we get a kick, but unfortunately results will be race-y.
time.Sleep(100 * time.Millisecond)
mgr.Kick("after-rollback")
},
Desc: rollbackDesc,
Path: "/api/index/idx0/count",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: afterRollbackCountResponseMatch,
},
}
testRESTHandlers(t, tests, router)
}
// Emits markdown docs of cbft's REST API.
func main() {
rand.Seed(0)
dataDir, _ := ioutil.TempDir("./tmp", "data")
defer os.RemoveAll(dataDir)
cfg := cbgt.NewCfgMem()
tags := []string(nil)
container := ""
weight := 1
extras := ""
bindHttp := "0.0.0.0:8095"
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
tags, container, weight, extras, bindHttp,
dataDir, "http://localhost:8091", nil)
sourceType := "nil"
sourceName := ""
sourceUUID := ""
sourceParams := ""
indexType := "blackhole"
indexName := "myFirstIndex"
indexParams := ""
prevIndexUUID := ""
mgr.Start("wanted")
err := mgr.CreateIndex(
sourceType, sourceName, sourceUUID, sourceParams,
indexType, indexName, indexParams,
cbgt.PlanParams{}, prevIndexUUID)
if err != nil {
log.Fatalf("could not create myFirstIndex, err: %v", err)
}
staticDir := ""
staticETag := ""
mr, _ := cbgt.NewMsgRing(ioutil.Discard, 1)
router, meta, err :=
cbft.NewRESTRouter(VERSION, mgr, staticDir, staticETag, mr)
if err != nil {
log.Panic(err)
}
mainCategoriesMap := map[string]bool{}
mainCategories := []string(nil)
subCategoriesMap := map[string]bool{}
subCategories := []string(nil)
paths := []string(nil)
for path := range meta {
paths = append(paths, path)
m := meta[path]
if m.Opts != nil {
category := m.Opts["_category"]
mainCategory, _, subCategory, _ := categoryParse(category)
if !mainCategoriesMap[mainCategory] {
mainCategoriesMap[mainCategory] = true
mainCategories = append(mainCategories, mainCategory)
}
if !subCategoriesMap[subCategory] {
subCategoriesMap[subCategory] = true
subCategories = append(subCategories, subCategory)
}
}
}
sort.Strings(mainCategories)
sort.Strings(subCategories)
sort.Strings(paths)
fmt.Printf("# API Reference\n\n")
for _, mainCategory := range mainCategories {
mainCategoryFirst := true
for _, subCategory := range subCategories {
subCategoryFirst := true
for _, path := range paths {
m := meta[path]
category := ""
if m.Opts != nil {
category = m.Opts["_category"]
}
mc, mcVis, sc, scVis := categoryParse(category)
if mc != mainCategory ||
sc != subCategory {
continue
}
if m.Opts != nil &&
m.Opts["_status"] == "private" {
continue
}
if mainCategoryFirst {
fmt.Printf("---\n\n# %s\n\n", mcVis)
}
mainCategoryFirst = false
if subCategoryFirst {
fmt.Printf("---\n\n## %s\n\n", scVis)
}
subCategoryFirst = false
pathParts := strings.Split(path, " ")
fmt.Printf("---\n\n")
fmt.Printf("%s `%s`\n\n", m.Method, pathParts[0])
if m.Opts != nil && m.Opts["_about"] != "" {
fmt.Printf("%s\n\n", m.Opts["_about"])
}
optNames := []string(nil)
for optName := range m.Opts {
if optName != "" && !strings.HasPrefix(optName, "_") {
optNames = append(optNames, optName)
}
}
sort.Strings(optNames)
for _, optName := range optNames {
fmt.Printf("**%s**: %s\n\n", optName, m.Opts[optName])
}
if m.Opts != nil && m.Opts[""] != "" {
fmt.Printf("%s\n\n", m.Opts[""])
}
if m.Method == "GET" && !skipSampleResponses[pathParts[0]] {
url := strings.Replace(pathParts[0],
"{indexName}", "myFirstIndex", 1)
res := sampleRequest(router, m.Method, url, nil)
if res != nil {
var j map[string]interface{}
err = json.Unmarshal(res, &j)
if err == nil {
s, err := json.MarshalIndent(j, " ", " ")
if err == nil {
fmt.Printf("Sample response:\n\n %s\n\n", s)
}
}
}
}
}
}
}
fmt.Printf("---\n\nCopyright (c) 2015 Couchbase, Inc.\n")
}