func TestNewRESTRouter(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
ring, err := cbgt.NewMsgRing(nil, 1)
cfg := cbgt.NewCfgMem()
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000",
emptyDir, "some-datasource", nil)
r, meta, err := NewRESTRouter("v0", mgr, emptyDir, "", ring,
AssetDir, Asset)
if r == nil || meta == nil || err != nil {
t.Errorf("expected no errors")
}
mgr = cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
[]string{"queryer", "anotherTag"},
"", 1, "", ":1000", emptyDir, "some-datasource", nil)
r, meta, err = NewRESTRouter("v0", mgr, emptyDir, "", ring,
AssetDir, Asset)
if r == nil || meta == nil || err != nil {
t.Errorf("expected no errors")
}
}
// MainUUID is a helper function for cmd-line tool developers, that
// reuses a previous "baseName.uuid" file from the dataDir if it
// exists, or generates a brand new UUID (and persists it).
func MainUUID(baseName, dataDir string) (string, error) {
uuid := cbgt.NewUUID()
uuidPath := dataDir + string(os.PathSeparator) + baseName + ".uuid"
uuidBuf, err := ioutil.ReadFile(uuidPath)
if err == nil {
uuid = strings.TrimSpace(string(uuidBuf))
if uuid == "" {
return "", fmt.Errorf("error: could not parse uuidPath: %s",
uuidPath)
}
log.Printf("main: manager uuid: %s", uuid)
log.Printf("main: manager uuid was reloaded")
} else {
log.Printf("main: manager uuid: %s", uuid)
log.Printf("main: manager uuid was generated")
}
err = ioutil.WriteFile(uuidPath, []byte(uuid), 0600)
if err != nil {
return "", fmt.Errorf("error: could not write uuidPath: %s\n"+
" Please check that your -data/-dataDir parameter (%q)\n"+
" is to a writable directory where %s can store\n"+
" index data.",
uuidPath, dataDir, baseName)
}
return uuid, nil
}
func TestMainStart(t *testing.T) {
mr, err := cbgt.NewMsgRing(os.Stderr, 1000)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
router, err := MainStart(nil, cbgt.NewUUID(), nil, "", 1, "", ":1000",
"bad data dir", "./static", "etag", "", "", mr, "k=v,k2=v2")
if router != nil || err == nil {
t.Errorf("expected empty server string to fail mainStart()")
}
router, err = MainStart(nil, cbgt.NewUUID(), nil, "", 1, "", ":1000",
"bad data dir", "./static", "etag", "bad server", "", mr, "")
if router != nil || err == nil {
t.Errorf("expected bad server string to fail mainStart()")
}
}
func TestManagerRestart(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
m := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil)
if err := m.Start("wanted"); err != nil {
t.Errorf("expected Manager.Start() to work, err: %v", err)
}
sourceParams := ""
if err := m.CreateIndex("primary", "default", "123", sourceParams,
"bleve", "foo", "", cbgt.PlanParams{},
"bad-prevIndexUUID"); err == nil {
t.Errorf("expected CreateIndex() err" +
" on attempted create-with-prevIndexUUID")
}
if err := m.CreateIndex("primary", "default", "123", sourceParams,
"bleve", "foo", "", cbgt.PlanParams{},
""); err != nil {
t.Errorf("expected CreateIndex() to work, err: %v", err)
}
if err := m.CreateIndex("primary", "default", "123", sourceParams,
"bleve", "foo", "", cbgt.PlanParams{},
"bad-prevIndexUUID"); err == nil {
t.Errorf("expected CreateIndex() err on update" +
" with wrong prevIndexUUID")
}
m.Kick("test0")
m.PlannerNOOP("test0")
feeds, pindexes := m.CurrentMaps()
if len(feeds) != 1 || len(pindexes) != 1 {
t.Errorf("expected to be 1 feed and 1 pindex,"+
" got feeds: %+v, pindexes: %+v",
feeds, pindexes)
}
for _, pindex := range pindexes {
pindex.Dest.Close()
if m.GetPIndex(pindex.Name) != pindex {
t.Errorf("expected GetPIndex() to match")
}
}
m2 := cbgt.NewManager(cbgt.VERSION, cfg, m.UUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil)
if err := m2.Start("wanted"); err != nil {
t.Errorf("expected reload Manager.Start() to work, err: %v", err)
}
m2.Kick("test2")
m2.PlannerNOOP("test2")
feeds, pindexes = m2.CurrentMaps()
if len(feeds) != 1 || len(pindexes) != 1 {
t.Errorf("expected to load 1 feed and 1 pindex,"+
" got feeds: %+v, pindexes: %+v",
feeds, pindexes)
}
}
func testPartitioning(t *testing.T,
sourceParams string,
planParams cbgt.PlanParams,
expectedNumPIndexes int,
expectedNumDests int,
andThen func(mgr *cbgt.Manager,
sf *cbgt.PrimaryFeed, pindexes map[string]*cbgt.PIndex)) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
if err := mgr.Start("wanted"); err != nil {
t.Errorf("expected Manager.Start() to work, err: %v", err)
}
if err := mgr.CreateIndex("primary",
"sourceName", "sourceUUID", sourceParams,
"bleve", "foo", "", planParams, ""); err != nil {
t.Errorf("expected CreateIndex() to work")
}
mgr.Kick("test")
mgr.PlannerNOOP("test")
mgr.JanitorNOOP("test")
feeds, pindexes := mgr.CurrentMaps()
if len(feeds) != 1 {
t.Errorf("expected to be 1 feed, got feeds: %+v", feeds)
}
if len(pindexes) != expectedNumPIndexes {
t.Errorf("expected to be %d pindex, got pindexes: %+v",
expectedNumPIndexes, pindexes)
}
var feed cbgt.Feed
for _, f := range feeds {
feed = f
}
sf, ok := feed.(*cbgt.PrimaryFeed)
if !ok || sf == nil {
t.Errorf("expected feed to be simple")
}
if len(sf.Dests()) != expectedNumDests {
t.Errorf("expected %d dests", expectedNumDests)
}
if andThen != nil {
andThen(mgr, sf, pindexes)
}
}
// updatePlanPIndexes_unlocked modifies the planPIndexes in/out param
// based on the indexDef/node/state/op params, and may return an error
// if the state transition is invalid.
func (r *Rebalancer) updatePlanPIndexes_unlocked(
planPIndexes *cbgt.PlanPIndexes, indexDef *cbgt.IndexDef,
pindex, node, state, op string) error {
planPIndex, err := r.getPlanPIndex_unlocked(planPIndexes, pindex)
if err != nil {
return err
}
canRead, canWrite :=
r.getNodePlanParamsReadWrite(indexDef, pindex, node)
if planPIndex.Nodes == nil {
planPIndex.Nodes = make(map[string]*cbgt.PlanPIndexNode)
}
priority := 0
if state == "replica" {
priority = len(planPIndex.Nodes)
}
if op == "add" {
if planPIndex.Nodes[node] != nil {
return fmt.Errorf("updatePlanPIndexes:"+
" planPIndex already exists,"+
" indexDef: %#v, pindex: %s,"+
" node: %s, state: %q, op: %s, planPIndex: %#v",
indexDef, pindex, node, state, op, planPIndex)
}
// TODO: Need to shift the other node priorities around?
planPIndex.Nodes[node] = &cbgt.PlanPIndexNode{
CanRead: canRead,
CanWrite: canWrite,
Priority: priority,
}
} else {
if planPIndex.Nodes[node] == nil {
return fmt.Errorf("updatePlanPIndexes:"+
" planPIndex missing,"+
" indexDef.Name: %s, pindex: %s,"+
" node: %s, state: %q, op: %s, planPIndex: %#v",
indexDef.Name, pindex, node, state, op, planPIndex)
}
if op == "del" {
// TODO: Need to shift the other node priorities around?
delete(planPIndex.Nodes, node)
} else {
// TODO: Need to shift the other node priorities around?
planPIndex.Nodes[node] = &cbgt.PlanPIndexNode{
CanRead: canRead,
CanWrite: canWrite,
Priority: priority,
}
}
}
planPIndex.UUID = cbgt.NewUUID()
planPIndexes.UUID = cbgt.NewUUID()
planPIndexes.ImplVersion = r.version
return nil
}
func TestHandlersForEmptyManager(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
err := mgr.Start("wanted")
if err != nil {
t.Errorf("expected start ok")
}
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
mr.Write([]byte("hello"))
mr.Write([]byte("world"))
mgr.AddEvent([]byte(`"fizz"`))
mgr.AddEvent([]byte(`"buzz"`))
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr,
AssetDir, Asset)
if err != nil || router == nil {
t.Errorf("no mux router")
}
tests := []*RESTHandlerTest{
{
Desc: "log on empty msg ring",
Path: "/api/log",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseBody: []byte(`{"messages":["hello","world"],"events":["fizz","buzz"]}`),
},
{
Desc: "cfg on empty manaager",
Path: "/api/cfg",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`"status":"ok"`: true,
`"indexDefs":null`: true,
`"nodeDefsKnown":{`: true,
`"nodeDefsWanted":{`: true,
`"planPIndexes":null`: true,
},
},
{
Desc: "cfg refresh on empty, unchanged manager",
Path: "/api/cfgRefresh",
Method: "POST",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok"}`: true,
},
},
{
Desc: "manager kick on empty, unchanged manager",
Path: "/api/managerKick",
Method: "POST",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok"}`: true,
},
},
{
Desc: "manager meta",
Path: "/api/managerMeta",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`"status":"ok"`: true,
`"startSamples":{`: true,
},
},
{
Desc: "feed stats when no feeds",
Path: "/api/stats",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{`: true,
`}`: true,
},
},
{
Desc: "list empty indexes",
Path: "/api/index",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseBody: []byte(`{"status":"ok","indexDefs":null}`),
},
{
Desc: "try to get a nonexistent index",
Path: "/api/index/NOT-AN-INDEX",
Method: "GET",
Params: nil,
Body: nil,
Status: 400,
ResponseBody: []byte(`index not found`),
},
{
Desc: "try to create a default index with no params",
Path: "/api/index/index-on-a-bad-server",
Method: "PUT",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`rest_create_index: indexType is required`: true,
},
},
{
Desc: "try to create a default index with no sourceType",
Path: "/api/index/index-on-a-bad-server",
Method: "PUT",
Params: url.Values{
"indexType": []string{"no-care"},
},
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`rest_create_index: sourceType is required`: true,
},
},
{
Desc: "try to create a default index with bad indexType",
Path: "/api/index/index-on-a-bad-server",
Method: "PUT",
Params: url.Values{
"indexType": []string{"no-care"},
"sourceType": []string{"couchbase"},
},
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`unknown indexType`: true,
},
},
{
Desc: "try to delete a nonexistent index when no indexes",
Path: "/api/index/NOT-AN-INDEX",
Method: "DELETE",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`no indexes`: true,
},
},
{
Desc: "try to count a nonexistent index when no indexes",
Path: "/api/index/NOT-AN-INDEX/count",
Method: "GET",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`could not get indexDefs`: true,
},
},
{
Desc: "try to query a nonexistent index when no indexes",
Path: "/api/index/NOT-AN-INDEX/query",
Method: "POST",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`could not get indexDefs`: true,
},
},
{
Desc: "create an index with bogus indexType",
Path: "/api/index/idxBogusIndexType",
Method: "PUT",
Params: url.Values{
"indexType": []string{"not-a-real-index-type"},
"sourceType": []string{"nil"},
},
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`error`: true,
},
},
{
Desc: "create a blackhole index",
Path: "/api/index/bh0",
Method: "PUT",
Params: url.Values{
"indexType": []string{"blackhole"},
"sourceType": []string{"nil"},
},
Body: nil,
Status: 200,
},
{
Desc: "create a blackhole index via body",
Path: "/api/index/bh1",
Method: "PUT",
Body: []byte(`{"type":"blackhole","sourceType":"nil"}`),
Status: 200,
},
{
Desc: "create a blackhole index, bad params",
Path: "/api/index/bh2",
Method: "PUT",
Body: []byte(`{"type":"blackhole","params":666,"sourceType":"nil"}`),
Status: 400,
ResponseMatch: map[string]bool{
`cannot unmarshal number into Go value`: true,
},
},
{
Desc: "create a blackhole index, bad params",
Path: "/api/index/bh2",
Method: "PUT",
Body: []byte(`{"type":"blackhole","params":"hi","sourceType":"nil"}`),
Status: 200,
},
{
Desc: "create a blackhole index, bad params",
Path: "/api/index/bh3",
Method: "PUT",
Body: []byte(`{"type":"blackhole","params":{},"sourceType":"nil"}`),
Status: 200,
},
{
Desc: "create a blackhole index, bad sourceParams",
Path: "/api/index/bh2s",
Method: "PUT",
Body: []byte(`{"type":"blackhole","sourceParams":666,"sourceType":"nil"}`),
Status: 400,
ResponseMatch: map[string]bool{
`cannot unmarshal number into Go value`: true,
},
},
{
Desc: "create a blackhole index, bad sourceParams",
Path: "/api/index/bh2s",
Method: "PUT",
Body: []byte(`{"type":"blackhole","sourceParams":"hi","sourceType":"nil"}`),
Status: 200,
},
{
Desc: "create a blackhole index, bad sourceParams",
Path: "/api/index/bh3s",
Method: "PUT",
Body: []byte(`{"type":"blackhole","sourceParams":{},"sourceType":"nil"}`),
Status: 200,
},
}
testRESTHandlers(t, tests, router)
}
func TestHandlersForRuntimeOps(t *testing.T) {
emptyDir, err := ioutil.TempDir("./tmp", "test")
if err != nil {
t.Errorf("tempdir err: %v", err)
}
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
err = mgr.Start("wanted")
if err != nil {
t.Errorf("expected no start err, got: %v", err)
}
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
mr.Write([]byte("hello"))
mr.Write([]byte("world"))
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr,
AssetDir, Asset)
if err != nil || router == nil {
t.Errorf("no mux router")
}
tests := []*RESTHandlerTest{
{
Path: "/api/runtime",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
"arch": true,
"go": true,
"GOMAXPROCS": true,
"GOROOT": true,
},
},
{
Path: "/api/runtime/args",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
// Actual production args are different from "go test" context.
},
},
{
Path: "/api/runtime/gc",
Method: "POST",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseBody: []byte(nil),
},
{
Path: "/api/runtime/statsMem",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
"Alloc": true,
"TotalAlloc": true,
},
},
{
Path: "/api/runtime/profile/cpu",
Method: "POST",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
"incorrect or missing secs parameter": true,
},
},
}
testRESTHandlers(t, tests, router)
}
func TestManagerIndexControl(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
m := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil)
if err := m.Start("wanted"); err != nil {
t.Errorf("expected Manager.Start() to work, err: %v", err)
}
sourceParams := ""
if err := m.CreateIndex("primary", "default", "123", sourceParams,
"bleve", "foo", "", cbgt.PlanParams{}, ""); err != nil {
t.Errorf("expected CreateIndex() to work, err: %v", err)
}
m.Kick("test0")
m.PlannerNOOP("test0")
err := m.IndexControl("foo", "wrong-uuid", "", "", "")
if err == nil {
t.Errorf("expected err on wrong UUID")
}
indexDefs, _, _ := cbgt.CfgGetIndexDefs(cfg)
npp := indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp != nil {
t.Errorf("expected nil npp")
}
err = m.IndexControl("foo", "", "", "", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] != nil {
t.Errorf("expected nil npp.sub")
}
err = m.IndexControl("foo", "", "disallow", "", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] == nil {
t.Errorf("expected npp.sub")
}
if npp[""].CanRead {
t.Errorf("expected CanRead false")
}
if !npp[""].CanWrite {
t.Errorf("expected CanWrite")
}
err = m.IndexControl("foo", "", "", "", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] == nil {
t.Errorf("expected npp.sub")
}
if npp[""].CanRead {
t.Errorf("expected CanRead false")
}
if !npp[""].CanWrite {
t.Errorf("expected CanWrite")
}
err = m.IndexControl("foo", "", "", "pause", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] == nil {
t.Errorf("expected npp.sub")
}
if npp[""].CanRead {
t.Errorf("expected CanRead false")
}
if npp[""].CanWrite {
t.Errorf("expected CanWrite false")
}
err = m.IndexControl("foo", "", "", "", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] == nil {
t.Errorf("expected npp.sub")
}
if npp[""].CanRead {
t.Errorf("expected CanRead false")
}
if npp[""].CanWrite {
t.Errorf("expected CanWrite false")
}
err = m.IndexControl("foo", "", "", "resume", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] == nil {
t.Errorf("expected npp.sub")
}
if npp[""].CanRead {
t.Errorf("expected CanRead false")
}
if !npp[""].CanWrite {
t.Errorf("expected CanWrite")
}
err = m.IndexControl("foo", "", "allow", "resume", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] != nil {
t.Errorf("expected nil npp.sub")
}
if indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen {
t.Errorf("expected not yet frozen")
}
err = m.IndexControl("foo", "", "", "", "freeze")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
if !indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen {
t.Errorf("expected frozen")
}
err = m.IndexControl("foo", "", "", "", "unfreeze")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
if indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen {
t.Errorf("expected not frozen")
}
}
func TestPreparePerms(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
mgr.Start("wanted")
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
if err != nil || router == nil {
t.Errorf("no mux router")
}
router.KeepContext = true // so we can see the mux vars
s := &stubDefinitionLookuper{
pindexes: testPIndexesByName,
defs: &cbgt.IndexDefs{
IndexDefs: testIndexDefsByName,
},
}
tests := []struct {
method string
uri string
body []byte
path string
perms []string
err error
}{
// case with valid perm not containing source
{
method: http.MethodGet,
uri: "/api/index",
path: "/api/index",
perms: []string{"cluster.bucket.fts!read"},
},
// case with valid index name
{
method: http.MethodGet,
uri: "/api/index/i1",
path: "/api/index/{indexName}",
perms: []string{"cluster.bucket[s1].fts!read"},
},
// case with invalid index name
{
method: http.MethodGet,
uri: "/api/index/x1",
path: "/api/index/{indexName}",
err: errIndexNotFound,
},
// case with invalid index name (actuall pindex name)
{
method: http.MethodGet,
uri: "/api/index/p1",
path: "/api/index/{indexName}",
err: errIndexNotFound,
},
// case with valid pindex name
{
method: http.MethodGet,
uri: "/api/pindex/p1",
path: "/api/pindex/{pindexName}",
perms: []string{"cluster.bucket[s3].fts!read"},
},
// case with invalid pindex name
{
method: http.MethodGet,
uri: "/api/pindex/y1",
path: "/api/pindex/{pindexName}",
err: errPIndexNotFound,
},
// case with invalid pindex name (actually index name)
{
method: http.MethodGet,
uri: "/api/pindex/i1",
path: "/api/pindex/{pindexName}",
err: errPIndexNotFound,
},
// case with valid alias, but this operation does NOT expand alias
{
method: http.MethodGet,
uri: "/api/index/a1",
path: "/api/index/{indexName}",
perms: []string{"cluster.bucket.fts!read"},
},
// case with valid alias, and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a1/count",
path: "/api/index/{indexName}/count",
perms: []string{"cluster.bucket[s1].fts!read"},
},
// case with valid alias (multi), and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a2/count",
path: "/api/index/{indexName}/count",
perms: []string{"cluster.bucket[s1].fts!read", "cluster.bucket[s2].fts!read"},
},
// case with valid alias (containing another alias),
// and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a3/count",
path: "/api/index/{indexName}/count",
perms: []string{"cluster.bucket[s1].fts!read", "cluster.bucket[s2].fts!read"},
},
// test special case for creating new index
{
method: http.MethodPut,
uri: "/api/index/anewone",
body: []byte(`{"type":"fulltext-index","sourceType":"couchbase","sourceName":"abucket"}`),
path: "/api/index/{indexName}",
perms: []string{"cluster.bucket[abucket].fts!write"},
},
}
for i, test := range tests {
var r io.Reader
if test.body != nil {
r = bytes.NewBuffer(test.body)
}
req, err := http.NewRequest(test.method, test.uri, r)
if err != nil {
t.Fatal(err)
}
// this actually executes things, which will usually fail
// which is unrelated to what we're testing
// but its the best i could do
record := httptest.NewRecorder()
router.ServeHTTP(record, req)
// set the request body again, as its been consumed :(
if test.body != nil {
req.Body = ioutil.NopCloser(bytes.NewBuffer(test.body))
}
actualPerms, err := preparePerms(s, req, test.method, test.path)
if err != test.err {
t.Errorf("test %d, expected err %v, got %v", i, test.err, err)
}
sort.Strings(actualPerms)
if !reflect.DeepEqual(actualPerms, test.perms) {
t.Errorf("test %d, expected %v, got %v", i, test.perms, actualPerms)
}
}
}
func TestSourceNamesFromReq(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
mgr.Start("wanted")
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
if err != nil || router == nil {
t.Errorf("no mux router")
}
router.KeepContext = true // so we can see the mux vars
s := &stubDefinitionLookuper{
pindexes: testPIndexesByName,
defs: &cbgt.IndexDefs{
IndexDefs: testIndexDefsByName,
},
}
tests := []struct {
method string
uri string
path string
sources []string
err error
}{
// case with valid index name
{
method: http.MethodGet,
uri: "/api/index/i1",
path: "/api/index/{indexName}",
sources: []string{"s1"},
},
// case with invalid index name
{
method: http.MethodGet,
uri: "/api/index/x1",
path: "/api/index/{indexName}",
err: errIndexNotFound,
},
// case with invalid index name (actuall pindex name)
{
method: http.MethodGet,
uri: "/api/index/p1",
path: "/api/index/{indexName}",
err: errIndexNotFound,
},
// case with valid pindex name
{
method: http.MethodGet,
uri: "/api/pindex/p1",
path: "/api/pindex/{pindexName}",
sources: []string{"s3"},
},
// case with invalid pindex name
{
method: http.MethodGet,
uri: "/api/pindex/y1",
path: "/api/pindex/{pindexName}",
err: errPIndexNotFound,
},
// case with invalid pindex name (actually index name)
{
method: http.MethodGet,
uri: "/api/pindex/i1",
path: "/api/pindex/{pindexName}",
err: errPIndexNotFound,
},
// case with valid alias, but this operation does NOT expand alias
{
method: http.MethodGet,
uri: "/api/index/a1",
path: "/api/index/{indexName}",
sources: []string{},
},
// case with valid alias, and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a1/count",
path: "/api/index/{indexName}/count",
sources: []string{"s1"},
},
// case with valid alias (multi), and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a2/count",
path: "/api/index/{indexName}/count",
sources: []string{"s1", "s2"},
},
// case with valid alias (containing another alias),
// and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a3/count",
path: "/api/index/{indexName}/count",
sources: []string{"s1", "s2"},
},
}
for i, test := range tests {
req, err := http.NewRequest(test.method, test.uri, nil)
if err != nil {
t.Fatal(err)
}
// this actually executes things, which will usually fail
// which is unrelated to what we're testing
// but its the best i could do
record := httptest.NewRecorder()
router.ServeHTTP(record, req)
actualNames, err := sourceNamesFromReq(s, req, test.method, test.path)
if err != test.err {
t.Errorf("test %d, expected err %v, got %v", i, test.err, err)
}
sort.Strings(actualNames)
if !reflect.DeepEqual(actualNames, test.sources) {
t.Errorf("test %d, expected %v, got %v", i, test.sources, actualNames)
}
}
}
func testHandlersWithOnePartitionPrimaryFeedPartialRollbackMoss(t *testing.T,
rollbackToSeq uint64,
rollbackDesc string,
afterRollbackCountResponseMatch map[string]bool) {
BlevePIndexAllowMossPrev := BlevePIndexAllowMoss
BlevePIndexAllowMoss = true
bleveConfigDefaultKVStorePrev := bleve.Config.DefaultKVStore
bleve.Config.DefaultKVStore = "mossStore"
bleveMossRegistryCollectionOptionsFTSPrev := bleveMoss.RegistryCollectionOptions["fts"]
bleveMoss.RegistryCollectionOptions["fts"] = moss.CollectionOptions{}
defer func() {
BlevePIndexAllowMoss = BlevePIndexAllowMossPrev
bleve.Config.DefaultKVStore = bleveConfigDefaultKVStorePrev
bleveMoss.RegistryCollectionOptions["fts"] = bleveMossRegistryCollectionOptionsFTSPrev
}()
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
mgr.Start("wanted")
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
if err != nil || router == nil {
t.Errorf("no mux router")
}
var pindex *cbgt.PIndex
var feed *cbgt.PrimaryFeed
waitForPersistence := func() {
for i := 0; i < 100; i++ {
stats := map[string]interface{}{
"num_recs_to_persist": float64(0),
}
err := addPIndexStats(pindex, stats)
if err != nil {
t.Errorf("expected nil addPIndexStats err, got: %v", err)
}
v, ok := stats["num_recs_to_persist"]
if ok {
nrtp, ok := v.(float64)
if ok && nrtp <= 0.0 {
return
}
}
time.Sleep(50 * time.Millisecond)
}
t.Errorf("persistence took too long!")
}
tests := []*RESTHandlerTest{
{
Desc: "create dest feed with 1 partition for rollback",
Path: "/api/index/idx0",
Method: "PUT",
Params: url.Values{
"indexType": []string{"fulltext-index"},
"indexParams": []string{
`{"store":{"mossStoreOptions":{"CompactionPercentage":100000.0}}}`,
}, // Never compact during this test.
"sourceType": []string{"primary"},
"sourceParams": []string{`{"numPartitions":1}`},
},
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok"}`: true,
},
After: func() {
time.Sleep(10 * time.Millisecond)
mgr.Kick("after-rollback")
feeds, pindexes := mgr.CurrentMaps()
if len(feeds) != 1 {
t.Errorf("expected to be 1 feed, got feeds: %+v", feeds)
}
if len(pindexes) != 1 {
t.Errorf("expected to be 1 pindex,"+
" got pindexes: %+v", pindexes)
}
for _, f := range feeds {
var ok bool
feed, ok = f.(*cbgt.PrimaryFeed)
if !ok {
t.Errorf("expected the 1 feed to be a PrimaryFeed")
}
}
for _, p := range pindexes {
pindex = p
}
},
},
{
Before: func() {
partition := "0"
snapStart := uint64(1)
snapEnd := uint64(10)
err = feed.SnapshotStart(partition, snapStart, snapEnd)
if err != nil {
t.Errorf("expected no err on snapshot-start")
}
},
Desc: "count idx0 0 when snapshot just started, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":0}`: true,
},
},
{
Before: func() {
partition := "0"
key := []byte("ss1-k1")
seq := uint64(1)
val := []byte(`{"foo":"bar","yow":"wow"}`)
err = feed.DataUpdate(partition, key, seq, val,
0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
if err != nil {
t.Errorf("expected no err on data-update")
}
},
Desc: "count idx0 should be 0 when got an update (doc created)" +
" but snapshot not ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":0}`: true,
},
},
{
Before: func() {
partition := "0"
key := []byte("ss1-k2")
seq := uint64(2)
val := []byte(`{"foo":"bing","yow":"wow"}`)
err = feed.DataUpdate(partition, key, seq, val,
0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
if err != nil {
t.Errorf("expected no err on data-update")
}
},
Desc: "count idx0 0 when got 2nd update (another doc created)" +
" but snapshot not ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":0}`: true,
},
},
{
Before: func() {
partition := "0"
snapStart := uint64(11)
snapEnd := uint64(20)
err = feed.SnapshotStart(partition, snapStart, snapEnd)
if err != nil {
t.Errorf("expected no err on snapshot-start")
}
waitForPersistence()
},
Desc: "count idx0 2 when 1st snapshot ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":2}`: true,
},
},
{
Before: func() {
partition := "0"
key := []byte("ss11-k11")
seq := uint64(11)
val := []byte(`{"foo":"baz","yow":"NOPE"}`)
err = feed.DataUpdate(partition, key, seq, val,
0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
if err != nil {
t.Errorf("expected no err on data-update")
}
},
Desc: "count idx0 should be 2 when got an update (doc created)" +
" in 2nd snapshot, 2nd snapshot not ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":2}`: true,
},
},
{
Before: func() {
partition := "0"
snapStart := uint64(21)
snapEnd := uint64(30)
err = feed.SnapshotStart(partition, snapStart, snapEnd)
if err != nil {
t.Errorf("expected no err on snapshot-start")
}
waitForPersistence()
},
Desc: "count idx0 3 when 2nd snapshot ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":3}`: true,
},
},
{
Before: func() {
partition := "0"
key := []byte("ss21-k21")
seq := uint64(21)
val := []byte(`{"foo":"baz","yow":"whoa"}`)
err = feed.DataUpdate(partition, key, seq, val,
0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
if err != nil {
t.Errorf("expected no err on data-update")
}
},
Desc: "count idx0 should be 3 when got an update (doc created)" +
" in 3rd snapshot, 3rd snapshot not ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":3}`: true,
},
},
{
Before: func() {
partition := "0"
snapStart := uint64(31)
snapEnd := uint64(40)
err = feed.SnapshotStart(partition, snapStart, snapEnd)
if err != nil {
t.Errorf("expected no err on snapshot-start")
}
waitForPersistence()
},
Desc: "count idx0 4 when 3rd snapshot ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":4}`: true,
},
},
{
// Rollback to a high seq number, so it's a no-op
// w.r.t. doing an actual rollback.
Before: func() {
partition := "0"
err = feed.Rollback(partition, rollbackToSeq)
if err != nil {
t.Errorf("expected no err on rollback, got: %v", err)
}
// NOTE: We might check right after rollback but before
// we get a kick, but unfortunately results will be race-y.
time.Sleep(100 * time.Millisecond)
mgr.Kick("after-rollback")
},
Desc: rollbackDesc,
Path: "/api/index/idx0/count",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: afterRollbackCountResponseMatch,
},
}
testRESTHandlers(t, tests, router)
}
// Emits markdown docs of cbft's REST API.
func main() {
rand.Seed(0)
dataDir, _ := ioutil.TempDir("./tmp", "data")
defer os.RemoveAll(dataDir)
cfg := cbgt.NewCfgMem()
tags := []string(nil)
container := ""
weight := 1
extras := ""
bindHttp := "0.0.0.0:8095"
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
tags, container, weight, extras, bindHttp,
dataDir, "http://localhost:8091", nil)
sourceType := "nil"
sourceName := ""
sourceUUID := ""
sourceParams := ""
indexType := "blackhole"
indexName := "myFirstIndex"
indexParams := ""
prevIndexUUID := ""
mgr.Start("wanted")
err := mgr.CreateIndex(
sourceType, sourceName, sourceUUID, sourceParams,
indexType, indexName, indexParams,
cbgt.PlanParams{}, prevIndexUUID)
if err != nil {
log.Fatalf("could not create myFirstIndex, err: %v", err)
}
staticDir := ""
staticETag := ""
mr, _ := cbgt.NewMsgRing(ioutil.Discard, 1)
router, meta, err :=
cbft.NewRESTRouter(VERSION, mgr, staticDir, staticETag, mr)
if err != nil {
log.Panic(err)
}
mainCategoriesMap := map[string]bool{}
mainCategories := []string(nil)
subCategoriesMap := map[string]bool{}
subCategories := []string(nil)
paths := []string(nil)
for path := range meta {
paths = append(paths, path)
m := meta[path]
if m.Opts != nil {
category := m.Opts["_category"]
mainCategory, _, subCategory, _ := categoryParse(category)
if !mainCategoriesMap[mainCategory] {
mainCategoriesMap[mainCategory] = true
mainCategories = append(mainCategories, mainCategory)
}
if !subCategoriesMap[subCategory] {
subCategoriesMap[subCategory] = true
subCategories = append(subCategories, subCategory)
}
}
}
sort.Strings(mainCategories)
sort.Strings(subCategories)
sort.Strings(paths)
fmt.Printf("# API Reference\n\n")
for _, mainCategory := range mainCategories {
mainCategoryFirst := true
for _, subCategory := range subCategories {
subCategoryFirst := true
for _, path := range paths {
m := meta[path]
category := ""
if m.Opts != nil {
category = m.Opts["_category"]
}
mc, mcVis, sc, scVis := categoryParse(category)
if mc != mainCategory ||
sc != subCategory {
continue
}
if m.Opts != nil &&
m.Opts["_status"] == "private" {
continue
}
if mainCategoryFirst {
fmt.Printf("---\n\n# %s\n\n", mcVis)
}
mainCategoryFirst = false
if subCategoryFirst {
fmt.Printf("---\n\n## %s\n\n", scVis)
}
subCategoryFirst = false
pathParts := strings.Split(path, " ")
fmt.Printf("---\n\n")
fmt.Printf("%s `%s`\n\n", m.Method, pathParts[0])
if m.Opts != nil && m.Opts["_about"] != "" {
fmt.Printf("%s\n\n", m.Opts["_about"])
}
optNames := []string(nil)
for optName := range m.Opts {
if optName != "" && !strings.HasPrefix(optName, "_") {
optNames = append(optNames, optName)
}
}
sort.Strings(optNames)
for _, optName := range optNames {
fmt.Printf("**%s**: %s\n\n", optName, m.Opts[optName])
}
if m.Opts != nil && m.Opts[""] != "" {
fmt.Printf("%s\n\n", m.Opts[""])
}
if m.Method == "GET" && !skipSampleResponses[pathParts[0]] {
url := strings.Replace(pathParts[0],
"{indexName}", "myFirstIndex", 1)
res := sampleRequest(router, m.Method, url, nil)
if res != nil {
var j map[string]interface{}
err = json.Unmarshal(res, &j)
if err == nil {
s, err := json.MarshalIndent(j, " ", " ")
if err == nil {
fmt.Printf("Sample response:\n\n %s\n\n", s)
}
}
}
}
}
}
}
fmt.Printf("---\n\nCopyright (c) 2015 Couchbase, Inc.\n")
}