func TestNewRESTRouter(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
ring, err := cbgt.NewMsgRing(nil, 1)
cfg := cbgt.NewCfgMem()
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000",
emptyDir, "some-datasource", nil)
r, meta, err := NewRESTRouter("v0", mgr, emptyDir, "", ring,
AssetDir, Asset)
if r == nil || meta == nil || err != nil {
t.Errorf("expected no errors")
}
mgr = cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
[]string{"queryer", "anotherTag"},
"", 1, "", ":1000", emptyDir, "some-datasource", nil)
r, meta, err = NewRESTRouter("v0", mgr, emptyDir, "", ring,
AssetDir, Asset)
if r == nil || meta == nil || err != nil {
t.Errorf("expected no errors")
}
}
func TestManagerRestart(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
m := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil)
if err := m.Start("wanted"); err != nil {
t.Errorf("expected Manager.Start() to work, err: %v", err)
}
sourceParams := ""
if err := m.CreateIndex("primary", "default", "123", sourceParams,
"bleve", "foo", "", cbgt.PlanParams{},
"bad-prevIndexUUID"); err == nil {
t.Errorf("expected CreateIndex() err" +
" on attempted create-with-prevIndexUUID")
}
if err := m.CreateIndex("primary", "default", "123", sourceParams,
"bleve", "foo", "", cbgt.PlanParams{},
""); err != nil {
t.Errorf("expected CreateIndex() to work, err: %v", err)
}
if err := m.CreateIndex("primary", "default", "123", sourceParams,
"bleve", "foo", "", cbgt.PlanParams{},
"bad-prevIndexUUID"); err == nil {
t.Errorf("expected CreateIndex() err on update" +
" with wrong prevIndexUUID")
}
m.Kick("test0")
m.PlannerNOOP("test0")
feeds, pindexes := m.CurrentMaps()
if len(feeds) != 1 || len(pindexes) != 1 {
t.Errorf("expected to be 1 feed and 1 pindex,"+
" got feeds: %+v, pindexes: %+v",
feeds, pindexes)
}
for _, pindex := range pindexes {
pindex.Dest.Close()
if m.GetPIndex(pindex.Name) != pindex {
t.Errorf("expected GetPIndex() to match")
}
}
m2 := cbgt.NewManager(cbgt.VERSION, cfg, m.UUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil)
if err := m2.Start("wanted"); err != nil {
t.Errorf("expected reload Manager.Start() to work, err: %v", err)
}
m2.Kick("test2")
m2.PlannerNOOP("test2")
feeds, pindexes = m2.CurrentMaps()
if len(feeds) != 1 || len(pindexes) != 1 {
t.Errorf("expected to load 1 feed and 1 pindex,"+
" got feeds: %+v, pindexes: %+v",
feeds, pindexes)
}
}
func testPartitioning(t *testing.T,
sourceParams string,
planParams cbgt.PlanParams,
expectedNumPIndexes int,
expectedNumDests int,
andThen func(mgr *cbgt.Manager,
sf *cbgt.PrimaryFeed, pindexes map[string]*cbgt.PIndex)) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
if err := mgr.Start("wanted"); err != nil {
t.Errorf("expected Manager.Start() to work, err: %v", err)
}
if err := mgr.CreateIndex("primary",
"sourceName", "sourceUUID", sourceParams,
"bleve", "foo", "", planParams, ""); err != nil {
t.Errorf("expected CreateIndex() to work")
}
mgr.Kick("test")
mgr.PlannerNOOP("test")
mgr.JanitorNOOP("test")
feeds, pindexes := mgr.CurrentMaps()
if len(feeds) != 1 {
t.Errorf("expected to be 1 feed, got feeds: %+v", feeds)
}
if len(pindexes) != expectedNumPIndexes {
t.Errorf("expected to be %d pindex, got pindexes: %+v",
expectedNumPIndexes, pindexes)
}
var feed cbgt.Feed
for _, f := range feeds {
feed = f
}
sf, ok := feed.(*cbgt.PrimaryFeed)
if !ok || sf == nil {
t.Errorf("expected feed to be simple")
}
if len(sf.Dests()) != expectedNumDests {
t.Errorf("expected %d dests", expectedNumDests)
}
if andThen != nil {
andThen(mgr, sf, pindexes)
}
}
func TestRebalance(t *testing.T) {
testDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(testDir)
nodeDir := func(node string) string {
d := testDir + string(os.PathSeparator) + node
os.MkdirAll(d, 0700)
return d
}
var mut sync.Mutex
httpGets := 0
httpGet := func(url string) (resp *http.Response, err error) {
mut.Lock()
httpGets++
mut.Unlock()
return &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(bytes.NewBuffer([]byte("{}"))),
}, nil
}
tests := []struct {
label string
ops string // Space separated "+a", "-x".
params map[string]string
expNodes string // Space separated list of nodes ("a"..."v").
expIndexes string // Space separated list of indxes ("x"..."z").
expStartErr bool
}{
{"1st node",
"+a", nil,
"a",
"",
true,
},
{"add 1st index x",
"+x", nil,
"a",
"x",
false,
},
{"add 2nd node b",
"+b", nil,
"a b",
"x",
false,
},
{"add 2nd index y",
"+y", nil,
"a b",
"x y",
false,
},
{"remove node b",
"-b", nil,
"a",
"x y",
false,
},
}
cfg := cbgt.NewCfgMem()
mgrs := map[string]*cbgt.Manager{}
var mgr0 *cbgt.Manager
server := "."
waitUntilEmptyCfgEvents := func(ch chan cbgt.CfgEvent) {
for {
select {
case <-ch:
default:
return
}
}
}
cfgEventsNodeDefsWanted := make(chan cbgt.CfgEvent, 100)
cfg.Subscribe(cbgt.NODE_DEFS_WANTED, cfgEventsNodeDefsWanted)
waitUntilEmptyCfgEventsNodeDefsWanted := func() {
waitUntilEmptyCfgEvents(cfgEventsNodeDefsWanted)
}
cfgEventsIndexDefs := make(chan cbgt.CfgEvent, 100)
cfg.Subscribe(cbgt.INDEX_DEFS_KEY, cfgEventsIndexDefs)
waitUntilEmptyCfgEventsIndexDefs := func() {
waitUntilEmptyCfgEvents(cfgEventsIndexDefs)
}
for testi, test := range tests {
log.Printf("testi: %d, label: %q", testi, test.label)
checkCurrStatesIndexes := false
nodesToRemove := []string(nil)
for opi, op := range strings.Split(test.ops, " ") {
log.Printf(" opi: %d, op: %s", opi, op)
name := op[1:2]
isIndexOp := name >= "x"
if isIndexOp {
indexName := name
log.Printf(" indexOp: %s, indexName: %s", op[0:1], indexName)
testCreateIndex(t, mgr0, indexName, test.params,
waitUntilEmptyCfgEventsIndexDefs)
checkCurrStatesIndexes = false
} else { // It's a node op.
nodeName := name
log.Printf(" nodeOp: %s, nodeName: %s", op[0:1], nodeName)
register := "wanted"
if op[0:1] == "-" {
register = "unknown"
}
if test.params["register"] != "" {
register = test.params["register"]
}
if test.params[nodeName+".register"] != "" {
register = test.params[nodeName+".register"]
}
if register == "unknown" {
nodesToRemove = append(nodesToRemove, nodeName)
// Delay actual unknown registration / removal
// until after rebalance finishes.
continue
}
waitUntilEmptyCfgEventsNodeDefsWanted()
mgr, err := startNodeManager(nodeDir(nodeName),
cfg, nodeName, register, test.params, server)
if err != nil || mgr == nil {
t.Errorf("expected no err, got: %#v", err)
}
if mgr0 == nil {
mgr0 = mgr
}
mgrs[nodeName] = mgr
mgr.Kick("kick")
waitUntilEmptyCfgEventsNodeDefsWanted()
checkCurrStatesIndexes = true
}
}
r, err := StartRebalance(cbgt.VERSION, cfg, ".",
nodesToRemove,
RebalanceOptions{
HttpGet: httpGet,
},
)
if (test.expStartErr && err == nil) ||
(!test.expStartErr && err != nil) {
t.Errorf("testi: %d, label: %q,"+
" expStartErr: %v, but got: %v",
testi, test.label,
test.expStartErr, err)
}
if err != nil || r == nil {
continue
}
progressCh := r.ProgressCh()
if progressCh == nil {
t.Errorf("expected progressCh")
}
err = nil
for progress := range progressCh {
if progress.Error != nil {
err = progress.Error
log.Printf("saw progress error: %#v\n", progress)
}
}
r.Stop()
if err != nil {
t.Errorf("expected no end err, got: %v", err)
}
for _, nodeToRemove := range nodesToRemove {
if mgrs[nodeToRemove] != nil {
mgrs[nodeToRemove].Stop()
delete(mgrs, nodeToRemove)
}
// TODO: Perhaps one day, the MCP will unregister the node;
// for now, we unregister it "manually".
if true {
waitUntilEmptyCfgEventsNodeDefsWanted()
mgr, err := startNodeManager(nodeDir(nodeToRemove),
cfg, nodeToRemove, "unknown", test.params, server)
if err != nil || mgr == nil {
t.Errorf("expected no err, got: %#v", err)
}
mgr.Kick("kick")
waitUntilEmptyCfgEventsNodeDefsWanted()
mgr.Stop()
}
}
endIndexDefs, endNodeDefs, endPlanPIndexes, endPlanPIndexesCAS, err :=
cbgt.PlannerGetPlan(cfg, cbgt.VERSION, "")
if err != nil ||
endIndexDefs == nil ||
endNodeDefs == nil ||
endPlanPIndexes == nil ||
endPlanPIndexesCAS == 0 {
t.Errorf("expected no err, got: %#v", err)
}
expNodes := strings.Split(test.expNodes, " ")
if len(expNodes) != len(endNodeDefs.NodeDefs) {
t.Errorf("len(expNodes) != len(endNodeDefs.NodeDefs), "+
" expNodes: %#v, endNodeDefs.NodeDefs: %#v",
expNodes, endNodeDefs.NodeDefs)
}
for _, expNode := range expNodes {
if endNodeDefs.NodeDefs[expNode] == nil {
t.Errorf("didn't find expNode: %s,"+
" expNodes: %#v, endNodeDefs.NodeDefs: %#v",
expNode, expNodes, endNodeDefs.NodeDefs)
}
}
expIndexes := strings.Split(test.expIndexes, " ")
r.Visit(func(
currStates CurrStates,
currSeqs CurrSeqs,
wantSeqs WantSeqs,
nextMoves map[string]*blance.NextMoves) {
if !checkCurrStatesIndexes {
return
}
if len(currStates) != len(expIndexes) {
t.Errorf("test.label: %s,"+
" len(expIndexes) != len(currStates), "+
" expIndexes: %#v, currStates: %#v, endIndexDefs: %#v",
test.label, expIndexes, currStates, endIndexDefs)
}
})
}
}
func TestHandlersForEmptyManager(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
err := mgr.Start("wanted")
if err != nil {
t.Errorf("expected start ok")
}
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
mr.Write([]byte("hello"))
mr.Write([]byte("world"))
mgr.AddEvent([]byte(`"fizz"`))
mgr.AddEvent([]byte(`"buzz"`))
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr,
AssetDir, Asset)
if err != nil || router == nil {
t.Errorf("no mux router")
}
tests := []*RESTHandlerTest{
{
Desc: "log on empty msg ring",
Path: "/api/log",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseBody: []byte(`{"messages":["hello","world"],"events":["fizz","buzz"]}`),
},
{
Desc: "cfg on empty manaager",
Path: "/api/cfg",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`"status":"ok"`: true,
`"indexDefs":null`: true,
`"nodeDefsKnown":{`: true,
`"nodeDefsWanted":{`: true,
`"planPIndexes":null`: true,
},
},
{
Desc: "cfg refresh on empty, unchanged manager",
Path: "/api/cfgRefresh",
Method: "POST",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok"}`: true,
},
},
{
Desc: "manager kick on empty, unchanged manager",
Path: "/api/managerKick",
Method: "POST",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok"}`: true,
},
},
{
Desc: "manager meta",
Path: "/api/managerMeta",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`"status":"ok"`: true,
`"startSamples":{`: true,
},
},
{
Desc: "feed stats when no feeds",
Path: "/api/stats",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{`: true,
`}`: true,
},
},
{
Desc: "list empty indexes",
Path: "/api/index",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseBody: []byte(`{"status":"ok","indexDefs":null}`),
},
{
Desc: "try to get a nonexistent index",
Path: "/api/index/NOT-AN-INDEX",
Method: "GET",
Params: nil,
Body: nil,
Status: 400,
ResponseBody: []byte(`index not found`),
},
{
Desc: "try to create a default index with no params",
Path: "/api/index/index-on-a-bad-server",
Method: "PUT",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`rest_create_index: indexType is required`: true,
},
},
{
Desc: "try to create a default index with no sourceType",
Path: "/api/index/index-on-a-bad-server",
Method: "PUT",
Params: url.Values{
"indexType": []string{"no-care"},
},
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`rest_create_index: sourceType is required`: true,
},
},
{
Desc: "try to create a default index with bad indexType",
Path: "/api/index/index-on-a-bad-server",
Method: "PUT",
Params: url.Values{
"indexType": []string{"no-care"},
"sourceType": []string{"couchbase"},
},
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`unknown indexType`: true,
},
},
{
Desc: "try to delete a nonexistent index when no indexes",
Path: "/api/index/NOT-AN-INDEX",
Method: "DELETE",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`no indexes`: true,
},
},
{
Desc: "try to count a nonexistent index when no indexes",
Path: "/api/index/NOT-AN-INDEX/count",
Method: "GET",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`could not get indexDefs`: true,
},
},
{
Desc: "try to query a nonexistent index when no indexes",
Path: "/api/index/NOT-AN-INDEX/query",
Method: "POST",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`could not get indexDefs`: true,
},
},
{
Desc: "create an index with bogus indexType",
Path: "/api/index/idxBogusIndexType",
Method: "PUT",
Params: url.Values{
"indexType": []string{"not-a-real-index-type"},
"sourceType": []string{"nil"},
},
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`error`: true,
},
},
{
Desc: "create a blackhole index",
Path: "/api/index/bh0",
Method: "PUT",
Params: url.Values{
"indexType": []string{"blackhole"},
"sourceType": []string{"nil"},
},
Body: nil,
Status: 200,
},
{
Desc: "create a blackhole index via body",
Path: "/api/index/bh1",
Method: "PUT",
Body: []byte(`{"type":"blackhole","sourceType":"nil"}`),
Status: 200,
},
{
Desc: "create a blackhole index, bad params",
Path: "/api/index/bh2",
Method: "PUT",
Body: []byte(`{"type":"blackhole","params":666,"sourceType":"nil"}`),
Status: 400,
ResponseMatch: map[string]bool{
`cannot unmarshal number into Go value`: true,
},
},
{
Desc: "create a blackhole index, bad params",
Path: "/api/index/bh2",
Method: "PUT",
Body: []byte(`{"type":"blackhole","params":"hi","sourceType":"nil"}`),
Status: 200,
},
{
Desc: "create a blackhole index, bad params",
Path: "/api/index/bh3",
Method: "PUT",
Body: []byte(`{"type":"blackhole","params":{},"sourceType":"nil"}`),
Status: 200,
},
{
Desc: "create a blackhole index, bad sourceParams",
Path: "/api/index/bh2s",
Method: "PUT",
Body: []byte(`{"type":"blackhole","sourceParams":666,"sourceType":"nil"}`),
Status: 400,
ResponseMatch: map[string]bool{
`cannot unmarshal number into Go value`: true,
},
},
{
Desc: "create a blackhole index, bad sourceParams",
Path: "/api/index/bh2s",
Method: "PUT",
Body: []byte(`{"type":"blackhole","sourceParams":"hi","sourceType":"nil"}`),
Status: 200,
},
{
Desc: "create a blackhole index, bad sourceParams",
Path: "/api/index/bh3s",
Method: "PUT",
Body: []byte(`{"type":"blackhole","sourceParams":{},"sourceType":"nil"}`),
Status: 200,
},
}
testRESTHandlers(t, tests, router)
}
func TestHandlersForRuntimeOps(t *testing.T) {
emptyDir, err := ioutil.TempDir("./tmp", "test")
if err != nil {
t.Errorf("tempdir err: %v", err)
}
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
err = mgr.Start("wanted")
if err != nil {
t.Errorf("expected no start err, got: %v", err)
}
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
mr.Write([]byte("hello"))
mr.Write([]byte("world"))
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr,
AssetDir, Asset)
if err != nil || router == nil {
t.Errorf("no mux router")
}
tests := []*RESTHandlerTest{
{
Path: "/api/runtime",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
"arch": true,
"go": true,
"GOMAXPROCS": true,
"GOROOT": true,
},
},
{
Path: "/api/runtime/args",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
// Actual production args are different from "go test" context.
},
},
{
Path: "/api/runtime/gc",
Method: "POST",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseBody: []byte(nil),
},
{
Path: "/api/runtime/statsMem",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
"Alloc": true,
"TotalAlloc": true,
},
},
{
Path: "/api/runtime/profile/cpu",
Method: "POST",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
"incorrect or missing secs parameter": true,
},
},
}
testRESTHandlers(t, tests, router)
}
func TestManagerIndexControl(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
m := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil)
if err := m.Start("wanted"); err != nil {
t.Errorf("expected Manager.Start() to work, err: %v", err)
}
sourceParams := ""
if err := m.CreateIndex("primary", "default", "123", sourceParams,
"bleve", "foo", "", cbgt.PlanParams{}, ""); err != nil {
t.Errorf("expected CreateIndex() to work, err: %v", err)
}
m.Kick("test0")
m.PlannerNOOP("test0")
err := m.IndexControl("foo", "wrong-uuid", "", "", "")
if err == nil {
t.Errorf("expected err on wrong UUID")
}
indexDefs, _, _ := cbgt.CfgGetIndexDefs(cfg)
npp := indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp != nil {
t.Errorf("expected nil npp")
}
err = m.IndexControl("foo", "", "", "", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] != nil {
t.Errorf("expected nil npp.sub")
}
err = m.IndexControl("foo", "", "disallow", "", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] == nil {
t.Errorf("expected npp.sub")
}
if npp[""].CanRead {
t.Errorf("expected CanRead false")
}
if !npp[""].CanWrite {
t.Errorf("expected CanWrite")
}
err = m.IndexControl("foo", "", "", "", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] == nil {
t.Errorf("expected npp.sub")
}
if npp[""].CanRead {
t.Errorf("expected CanRead false")
}
if !npp[""].CanWrite {
t.Errorf("expected CanWrite")
}
err = m.IndexControl("foo", "", "", "pause", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] == nil {
t.Errorf("expected npp.sub")
}
if npp[""].CanRead {
t.Errorf("expected CanRead false")
}
if npp[""].CanWrite {
t.Errorf("expected CanWrite false")
}
err = m.IndexControl("foo", "", "", "", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] == nil {
t.Errorf("expected npp.sub")
}
if npp[""].CanRead {
t.Errorf("expected CanRead false")
}
if npp[""].CanWrite {
t.Errorf("expected CanWrite false")
}
err = m.IndexControl("foo", "", "", "resume", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] == nil {
t.Errorf("expected npp.sub")
}
if npp[""].CanRead {
t.Errorf("expected CanRead false")
}
if !npp[""].CanWrite {
t.Errorf("expected CanWrite")
}
err = m.IndexControl("foo", "", "allow", "resume", "")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
if npp == nil {
t.Errorf("expected npp")
}
if npp[""] != nil {
t.Errorf("expected nil npp.sub")
}
if indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen {
t.Errorf("expected not yet frozen")
}
err = m.IndexControl("foo", "", "", "", "freeze")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
if !indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen {
t.Errorf("expected frozen")
}
err = m.IndexControl("foo", "", "", "", "unfreeze")
if err != nil {
t.Errorf("expected ok")
}
indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
if indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen {
t.Errorf("expected not frozen")
}
}
func TestPreparePerms(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
mgr.Start("wanted")
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
if err != nil || router == nil {
t.Errorf("no mux router")
}
router.KeepContext = true // so we can see the mux vars
s := &stubDefinitionLookuper{
pindexes: testPIndexesByName,
defs: &cbgt.IndexDefs{
IndexDefs: testIndexDefsByName,
},
}
tests := []struct {
method string
uri string
body []byte
path string
perms []string
err error
}{
// case with valid perm not containing source
{
method: http.MethodGet,
uri: "/api/index",
path: "/api/index",
perms: []string{"cluster.bucket.fts!read"},
},
// case with valid index name
{
method: http.MethodGet,
uri: "/api/index/i1",
path: "/api/index/{indexName}",
perms: []string{"cluster.bucket[s1].fts!read"},
},
// case with invalid index name
{
method: http.MethodGet,
uri: "/api/index/x1",
path: "/api/index/{indexName}",
err: errIndexNotFound,
},
// case with invalid index name (actuall pindex name)
{
method: http.MethodGet,
uri: "/api/index/p1",
path: "/api/index/{indexName}",
err: errIndexNotFound,
},
// case with valid pindex name
{
method: http.MethodGet,
uri: "/api/pindex/p1",
path: "/api/pindex/{pindexName}",
perms: []string{"cluster.bucket[s3].fts!read"},
},
// case with invalid pindex name
{
method: http.MethodGet,
uri: "/api/pindex/y1",
path: "/api/pindex/{pindexName}",
err: errPIndexNotFound,
},
// case with invalid pindex name (actually index name)
{
method: http.MethodGet,
uri: "/api/pindex/i1",
path: "/api/pindex/{pindexName}",
err: errPIndexNotFound,
},
// case with valid alias, but this operation does NOT expand alias
{
method: http.MethodGet,
uri: "/api/index/a1",
path: "/api/index/{indexName}",
perms: []string{"cluster.bucket.fts!read"},
},
// case with valid alias, and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a1/count",
path: "/api/index/{indexName}/count",
perms: []string{"cluster.bucket[s1].fts!read"},
},
// case with valid alias (multi), and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a2/count",
path: "/api/index/{indexName}/count",
perms: []string{"cluster.bucket[s1].fts!read", "cluster.bucket[s2].fts!read"},
},
// case with valid alias (containing another alias),
// and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a3/count",
path: "/api/index/{indexName}/count",
perms: []string{"cluster.bucket[s1].fts!read", "cluster.bucket[s2].fts!read"},
},
// test special case for creating new index
{
method: http.MethodPut,
uri: "/api/index/anewone",
body: []byte(`{"type":"fulltext-index","sourceType":"couchbase","sourceName":"abucket"}`),
path: "/api/index/{indexName}",
perms: []string{"cluster.bucket[abucket].fts!write"},
},
}
for i, test := range tests {
var r io.Reader
if test.body != nil {
r = bytes.NewBuffer(test.body)
}
req, err := http.NewRequest(test.method, test.uri, r)
if err != nil {
t.Fatal(err)
}
// this actually executes things, which will usually fail
// which is unrelated to what we're testing
// but its the best i could do
record := httptest.NewRecorder()
router.ServeHTTP(record, req)
// set the request body again, as its been consumed :(
if test.body != nil {
req.Body = ioutil.NopCloser(bytes.NewBuffer(test.body))
}
actualPerms, err := preparePerms(s, req, test.method, test.path)
if err != test.err {
t.Errorf("test %d, expected err %v, got %v", i, test.err, err)
}
sort.Strings(actualPerms)
if !reflect.DeepEqual(actualPerms, test.perms) {
t.Errorf("test %d, expected %v, got %v", i, test.perms, actualPerms)
}
}
}
func TestSourceNamesFromReq(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
mgr.Start("wanted")
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
if err != nil || router == nil {
t.Errorf("no mux router")
}
router.KeepContext = true // so we can see the mux vars
s := &stubDefinitionLookuper{
pindexes: testPIndexesByName,
defs: &cbgt.IndexDefs{
IndexDefs: testIndexDefsByName,
},
}
tests := []struct {
method string
uri string
path string
sources []string
err error
}{
// case with valid index name
{
method: http.MethodGet,
uri: "/api/index/i1",
path: "/api/index/{indexName}",
sources: []string{"s1"},
},
// case with invalid index name
{
method: http.MethodGet,
uri: "/api/index/x1",
path: "/api/index/{indexName}",
err: errIndexNotFound,
},
// case with invalid index name (actuall pindex name)
{
method: http.MethodGet,
uri: "/api/index/p1",
path: "/api/index/{indexName}",
err: errIndexNotFound,
},
// case with valid pindex name
{
method: http.MethodGet,
uri: "/api/pindex/p1",
path: "/api/pindex/{pindexName}",
sources: []string{"s3"},
},
// case with invalid pindex name
{
method: http.MethodGet,
uri: "/api/pindex/y1",
path: "/api/pindex/{pindexName}",
err: errPIndexNotFound,
},
// case with invalid pindex name (actually index name)
{
method: http.MethodGet,
uri: "/api/pindex/i1",
path: "/api/pindex/{pindexName}",
err: errPIndexNotFound,
},
// case with valid alias, but this operation does NOT expand alias
{
method: http.MethodGet,
uri: "/api/index/a1",
path: "/api/index/{indexName}",
sources: []string{},
},
// case with valid alias, and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a1/count",
path: "/api/index/{indexName}/count",
sources: []string{"s1"},
},
// case with valid alias (multi), and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a2/count",
path: "/api/index/{indexName}/count",
sources: []string{"s1", "s2"},
},
// case with valid alias (containing another alias),
// and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a3/count",
path: "/api/index/{indexName}/count",
sources: []string{"s1", "s2"},
},
}
for i, test := range tests {
req, err := http.NewRequest(test.method, test.uri, nil)
if err != nil {
t.Fatal(err)
}
// this actually executes things, which will usually fail
// which is unrelated to what we're testing
// but its the best i could do
record := httptest.NewRecorder()
router.ServeHTTP(record, req)
actualNames, err := sourceNamesFromReq(s, req, test.method, test.path)
if err != test.err {
t.Errorf("test %d, expected err %v, got %v", i, test.err, err)
}
sort.Strings(actualNames)
if !reflect.DeepEqual(actualNames, test.sources) {
t.Errorf("test %d, expected %v, got %v", i, test.sources, actualNames)
}
}
}
func testHandlersWithOnePartitionPrimaryFeedPartialRollbackMoss(t *testing.T,
rollbackToSeq uint64,
rollbackDesc string,
afterRollbackCountResponseMatch map[string]bool) {
BlevePIndexAllowMossPrev := BlevePIndexAllowMoss
BlevePIndexAllowMoss = true
bleveConfigDefaultKVStorePrev := bleve.Config.DefaultKVStore
bleve.Config.DefaultKVStore = "mossStore"
bleveMossRegistryCollectionOptionsFTSPrev := bleveMoss.RegistryCollectionOptions["fts"]
bleveMoss.RegistryCollectionOptions["fts"] = moss.CollectionOptions{}
defer func() {
BlevePIndexAllowMoss = BlevePIndexAllowMossPrev
bleve.Config.DefaultKVStore = bleveConfigDefaultKVStorePrev
bleveMoss.RegistryCollectionOptions["fts"] = bleveMossRegistryCollectionOptionsFTSPrev
}()
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
mgr.Start("wanted")
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
if err != nil || router == nil {
t.Errorf("no mux router")
}
var pindex *cbgt.PIndex
var feed *cbgt.PrimaryFeed
waitForPersistence := func() {
for i := 0; i < 100; i++ {
stats := map[string]interface{}{
"num_recs_to_persist": float64(0),
}
err := addPIndexStats(pindex, stats)
if err != nil {
t.Errorf("expected nil addPIndexStats err, got: %v", err)
}
v, ok := stats["num_recs_to_persist"]
if ok {
nrtp, ok := v.(float64)
if ok && nrtp <= 0.0 {
return
}
}
time.Sleep(50 * time.Millisecond)
}
t.Errorf("persistence took too long!")
}
tests := []*RESTHandlerTest{
{
Desc: "create dest feed with 1 partition for rollback",
Path: "/api/index/idx0",
Method: "PUT",
Params: url.Values{
"indexType": []string{"fulltext-index"},
"indexParams": []string{
`{"store":{"mossStoreOptions":{"CompactionPercentage":100000.0}}}`,
}, // Never compact during this test.
"sourceType": []string{"primary"},
"sourceParams": []string{`{"numPartitions":1}`},
},
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok"}`: true,
},
After: func() {
time.Sleep(10 * time.Millisecond)
mgr.Kick("after-rollback")
feeds, pindexes := mgr.CurrentMaps()
if len(feeds) != 1 {
t.Errorf("expected to be 1 feed, got feeds: %+v", feeds)
}
if len(pindexes) != 1 {
t.Errorf("expected to be 1 pindex,"+
" got pindexes: %+v", pindexes)
}
for _, f := range feeds {
var ok bool
feed, ok = f.(*cbgt.PrimaryFeed)
if !ok {
t.Errorf("expected the 1 feed to be a PrimaryFeed")
}
}
for _, p := range pindexes {
pindex = p
}
},
},
{
Before: func() {
partition := "0"
snapStart := uint64(1)
snapEnd := uint64(10)
err = feed.SnapshotStart(partition, snapStart, snapEnd)
if err != nil {
t.Errorf("expected no err on snapshot-start")
}
},
Desc: "count idx0 0 when snapshot just started, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":0}`: true,
},
},
{
Before: func() {
partition := "0"
key := []byte("ss1-k1")
seq := uint64(1)
val := []byte(`{"foo":"bar","yow":"wow"}`)
err = feed.DataUpdate(partition, key, seq, val,
0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
if err != nil {
t.Errorf("expected no err on data-update")
}
},
Desc: "count idx0 should be 0 when got an update (doc created)" +
" but snapshot not ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":0}`: true,
},
},
{
Before: func() {
partition := "0"
key := []byte("ss1-k2")
seq := uint64(2)
val := []byte(`{"foo":"bing","yow":"wow"}`)
err = feed.DataUpdate(partition, key, seq, val,
0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
if err != nil {
t.Errorf("expected no err on data-update")
}
},
Desc: "count idx0 0 when got 2nd update (another doc created)" +
" but snapshot not ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":0}`: true,
},
},
{
Before: func() {
partition := "0"
snapStart := uint64(11)
snapEnd := uint64(20)
err = feed.SnapshotStart(partition, snapStart, snapEnd)
if err != nil {
t.Errorf("expected no err on snapshot-start")
}
waitForPersistence()
},
Desc: "count idx0 2 when 1st snapshot ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":2}`: true,
},
},
{
Before: func() {
partition := "0"
key := []byte("ss11-k11")
seq := uint64(11)
val := []byte(`{"foo":"baz","yow":"NOPE"}`)
err = feed.DataUpdate(partition, key, seq, val,
0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
if err != nil {
t.Errorf("expected no err on data-update")
}
},
Desc: "count idx0 should be 2 when got an update (doc created)" +
" in 2nd snapshot, 2nd snapshot not ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":2}`: true,
},
},
{
Before: func() {
partition := "0"
snapStart := uint64(21)
snapEnd := uint64(30)
err = feed.SnapshotStart(partition, snapStart, snapEnd)
if err != nil {
t.Errorf("expected no err on snapshot-start")
}
waitForPersistence()
},
Desc: "count idx0 3 when 2nd snapshot ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":3}`: true,
},
},
{
Before: func() {
partition := "0"
key := []byte("ss21-k21")
seq := uint64(21)
val := []byte(`{"foo":"baz","yow":"whoa"}`)
err = feed.DataUpdate(partition, key, seq, val,
0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
if err != nil {
t.Errorf("expected no err on data-update")
}
},
Desc: "count idx0 should be 3 when got an update (doc created)" +
" in 3rd snapshot, 3rd snapshot not ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":3}`: true,
},
},
{
Before: func() {
partition := "0"
snapStart := uint64(31)
snapEnd := uint64(40)
err = feed.SnapshotStart(partition, snapStart, snapEnd)
if err != nil {
t.Errorf("expected no err on snapshot-start")
}
waitForPersistence()
},
Desc: "count idx0 4 when 3rd snapshot ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":4}`: true,
},
},
{
// Rollback to a high seq number, so it's a no-op
// w.r.t. doing an actual rollback.
Before: func() {
partition := "0"
err = feed.Rollback(partition, rollbackToSeq)
if err != nil {
t.Errorf("expected no err on rollback, got: %v", err)
}
// NOTE: We might check right after rollback but before
// we get a kick, but unfortunately results will be race-y.
time.Sleep(100 * time.Millisecond)
mgr.Kick("after-rollback")
},
Desc: rollbackDesc,
Path: "/api/index/idx0/count",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: afterRollbackCountResponseMatch,
},
}
testRESTHandlers(t, tests, router)
}
// Emits markdown docs of cbft's REST API.
func main() {
rand.Seed(0)
dataDir, _ := ioutil.TempDir("./tmp", "data")
defer os.RemoveAll(dataDir)
cfg := cbgt.NewCfgMem()
tags := []string(nil)
container := ""
weight := 1
extras := ""
bindHttp := "0.0.0.0:8095"
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
tags, container, weight, extras, bindHttp,
dataDir, "http://localhost:8091", nil)
sourceType := "nil"
sourceName := ""
sourceUUID := ""
sourceParams := ""
indexType := "blackhole"
indexName := "myFirstIndex"
indexParams := ""
prevIndexUUID := ""
mgr.Start("wanted")
err := mgr.CreateIndex(
sourceType, sourceName, sourceUUID, sourceParams,
indexType, indexName, indexParams,
cbgt.PlanParams{}, prevIndexUUID)
if err != nil {
log.Fatalf("could not create myFirstIndex, err: %v", err)
}
staticDir := ""
staticETag := ""
mr, _ := cbgt.NewMsgRing(ioutil.Discard, 1)
router, meta, err :=
cbft.NewRESTRouter(VERSION, mgr, staticDir, staticETag, mr)
if err != nil {
log.Panic(err)
}
mainCategoriesMap := map[string]bool{}
mainCategories := []string(nil)
subCategoriesMap := map[string]bool{}
subCategories := []string(nil)
paths := []string(nil)
for path := range meta {
paths = append(paths, path)
m := meta[path]
if m.Opts != nil {
category := m.Opts["_category"]
mainCategory, _, subCategory, _ := categoryParse(category)
if !mainCategoriesMap[mainCategory] {
mainCategoriesMap[mainCategory] = true
mainCategories = append(mainCategories, mainCategory)
}
if !subCategoriesMap[subCategory] {
subCategoriesMap[subCategory] = true
subCategories = append(subCategories, subCategory)
}
}
}
sort.Strings(mainCategories)
sort.Strings(subCategories)
sort.Strings(paths)
fmt.Printf("# API Reference\n\n")
for _, mainCategory := range mainCategories {
mainCategoryFirst := true
for _, subCategory := range subCategories {
subCategoryFirst := true
for _, path := range paths {
m := meta[path]
category := ""
if m.Opts != nil {
category = m.Opts["_category"]
}
mc, mcVis, sc, scVis := categoryParse(category)
if mc != mainCategory ||
sc != subCategory {
continue
}
if m.Opts != nil &&
m.Opts["_status"] == "private" {
continue
}
if mainCategoryFirst {
fmt.Printf("---\n\n# %s\n\n", mcVis)
}
mainCategoryFirst = false
if subCategoryFirst {
fmt.Printf("---\n\n## %s\n\n", scVis)
}
subCategoryFirst = false
pathParts := strings.Split(path, " ")
fmt.Printf("---\n\n")
fmt.Printf("%s `%s`\n\n", m.Method, pathParts[0])
if m.Opts != nil && m.Opts["_about"] != "" {
fmt.Printf("%s\n\n", m.Opts["_about"])
}
optNames := []string(nil)
for optName := range m.Opts {
if optName != "" && !strings.HasPrefix(optName, "_") {
optNames = append(optNames, optName)
}
}
sort.Strings(optNames)
for _, optName := range optNames {
fmt.Printf("**%s**: %s\n\n", optName, m.Opts[optName])
}
if m.Opts != nil && m.Opts[""] != "" {
fmt.Printf("%s\n\n", m.Opts[""])
}
if m.Method == "GET" && !skipSampleResponses[pathParts[0]] {
url := strings.Replace(pathParts[0],
"{indexName}", "myFirstIndex", 1)
res := sampleRequest(router, m.Method, url, nil)
if res != nil {
var j map[string]interface{}
err = json.Unmarshal(res, &j)
if err == nil {
s, err := json.MarshalIndent(j, " ", " ")
if err == nil {
fmt.Printf("Sample response:\n\n %s\n\n", s)
}
}
}
}
}
}
}
fmt.Printf("---\n\nCopyright (c) 2015 Couchbase, Inc.\n")
}