Esempio n. 1
0
func TestNewRESTRouter(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	ring, err := cbgt.NewMsgRing(nil, 1)

	cfg := cbgt.NewCfgMem()
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000",
		emptyDir, "some-datasource", nil)
	r, meta, err := NewRESTRouter("v0", mgr, emptyDir, "", ring,
		AssetDir, Asset)
	if r == nil || meta == nil || err != nil {
		t.Errorf("expected no errors")
	}

	mgr = cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		[]string{"queryer", "anotherTag"},
		"", 1, "", ":1000", emptyDir, "some-datasource", nil)
	r, meta, err = NewRESTRouter("v0", mgr, emptyDir, "", ring,
		AssetDir, Asset)
	if r == nil || meta == nil || err != nil {
		t.Errorf("expected no errors")
	}
}
Esempio n. 2
0
func TestManagerRestart(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	m := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil)
	if err := m.Start("wanted"); err != nil {
		t.Errorf("expected Manager.Start() to work, err: %v", err)
	}
	sourceParams := ""
	if err := m.CreateIndex("primary", "default", "123", sourceParams,
		"bleve", "foo", "", cbgt.PlanParams{},
		"bad-prevIndexUUID"); err == nil {
		t.Errorf("expected CreateIndex() err" +
			" on attempted create-with-prevIndexUUID")
	}
	if err := m.CreateIndex("primary", "default", "123", sourceParams,
		"bleve", "foo", "", cbgt.PlanParams{},
		""); err != nil {
		t.Errorf("expected CreateIndex() to work, err: %v", err)
	}
	if err := m.CreateIndex("primary", "default", "123", sourceParams,
		"bleve", "foo", "", cbgt.PlanParams{},
		"bad-prevIndexUUID"); err == nil {
		t.Errorf("expected CreateIndex() err on update" +
			" with wrong prevIndexUUID")
	}
	m.Kick("test0")
	m.PlannerNOOP("test0")
	feeds, pindexes := m.CurrentMaps()
	if len(feeds) != 1 || len(pindexes) != 1 {
		t.Errorf("expected to be 1 feed and 1 pindex,"+
			" got feeds: %+v, pindexes: %+v",
			feeds, pindexes)
	}
	for _, pindex := range pindexes {
		pindex.Dest.Close()
		if m.GetPIndex(pindex.Name) != pindex {
			t.Errorf("expected GetPIndex() to match")
		}
	}

	m2 := cbgt.NewManager(cbgt.VERSION, cfg, m.UUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil)
	if err := m2.Start("wanted"); err != nil {
		t.Errorf("expected reload Manager.Start() to work, err: %v", err)
	}
	m2.Kick("test2")
	m2.PlannerNOOP("test2")
	feeds, pindexes = m2.CurrentMaps()
	if len(feeds) != 1 || len(pindexes) != 1 {
		t.Errorf("expected to load 1 feed and 1 pindex,"+
			" got feeds: %+v, pindexes: %+v",
			feeds, pindexes)
	}
}
Esempio n. 3
0
func testPartitioning(t *testing.T,
	sourceParams string,
	planParams cbgt.PlanParams,
	expectedNumPIndexes int,
	expectedNumDests int,
	andThen func(mgr *cbgt.Manager,
		sf *cbgt.PrimaryFeed, pindexes map[string]*cbgt.PIndex)) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	meh := &TestMEH{}
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
	if err := mgr.Start("wanted"); err != nil {
		t.Errorf("expected Manager.Start() to work, err: %v", err)
	}

	if err := mgr.CreateIndex("primary",
		"sourceName", "sourceUUID", sourceParams,
		"bleve", "foo", "", planParams, ""); err != nil {
		t.Errorf("expected CreateIndex() to work")
	}

	mgr.Kick("test")
	mgr.PlannerNOOP("test")
	mgr.JanitorNOOP("test")
	feeds, pindexes := mgr.CurrentMaps()
	if len(feeds) != 1 {
		t.Errorf("expected to be 1 feed, got feeds: %+v", feeds)
	}
	if len(pindexes) != expectedNumPIndexes {
		t.Errorf("expected to be %d pindex, got pindexes: %+v",
			expectedNumPIndexes, pindexes)
	}
	var feed cbgt.Feed
	for _, f := range feeds {
		feed = f
	}
	sf, ok := feed.(*cbgt.PrimaryFeed)
	if !ok || sf == nil {
		t.Errorf("expected feed to be simple")
	}
	if len(sf.Dests()) != expectedNumDests {
		t.Errorf("expected %d dests", expectedNumDests)
	}

	if andThen != nil {
		andThen(mgr, sf, pindexes)
	}
}
Esempio n. 4
0
func TestRebalance(t *testing.T) {
	testDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(testDir)

	nodeDir := func(node string) string {
		d := testDir + string(os.PathSeparator) + node
		os.MkdirAll(d, 0700)
		return d
	}

	var mut sync.Mutex

	httpGets := 0
	httpGet := func(url string) (resp *http.Response, err error) {
		mut.Lock()
		httpGets++
		mut.Unlock()

		return &http.Response{
			StatusCode: 200,
			Body:       ioutil.NopCloser(bytes.NewBuffer([]byte("{}"))),
		}, nil
	}

	tests := []struct {
		label       string
		ops         string // Space separated "+a", "-x".
		params      map[string]string
		expNodes    string // Space separated list of nodes ("a"..."v").
		expIndexes  string // Space separated list of indxes ("x"..."z").
		expStartErr bool
	}{
		{"1st node",
			"+a", nil,
			"a",
			"",
			true,
		},
		{"add 1st index x",
			"+x", nil,
			"a",
			"x",
			false,
		},
		{"add 2nd node b",
			"+b", nil,
			"a b",
			"x",
			false,
		},
		{"add 2nd index y",
			"+y", nil,
			"a b",
			"x y",
			false,
		},
		{"remove node b",
			"-b", nil,
			"a",
			"x y",
			false,
		},
	}

	cfg := cbgt.NewCfgMem()

	mgrs := map[string]*cbgt.Manager{}

	var mgr0 *cbgt.Manager

	server := "."

	waitUntilEmptyCfgEvents := func(ch chan cbgt.CfgEvent) {
		for {
			select {
			case <-ch:
			default:
				return
			}
		}
	}

	cfgEventsNodeDefsWanted := make(chan cbgt.CfgEvent, 100)
	cfg.Subscribe(cbgt.NODE_DEFS_WANTED, cfgEventsNodeDefsWanted)

	waitUntilEmptyCfgEventsNodeDefsWanted := func() {
		waitUntilEmptyCfgEvents(cfgEventsNodeDefsWanted)
	}

	cfgEventsIndexDefs := make(chan cbgt.CfgEvent, 100)
	cfg.Subscribe(cbgt.INDEX_DEFS_KEY, cfgEventsIndexDefs)

	waitUntilEmptyCfgEventsIndexDefs := func() {
		waitUntilEmptyCfgEvents(cfgEventsIndexDefs)
	}

	for testi, test := range tests {
		log.Printf("testi: %d, label: %q", testi, test.label)

		checkCurrStatesIndexes := false

		nodesToRemove := []string(nil)

		for opi, op := range strings.Split(test.ops, " ") {
			log.Printf(" opi: %d, op: %s", opi, op)

			name := op[1:2]

			isIndexOp := name >= "x"
			if isIndexOp {
				indexName := name
				log.Printf(" indexOp: %s, indexName: %s", op[0:1], indexName)

				testCreateIndex(t, mgr0, indexName, test.params,
					waitUntilEmptyCfgEventsIndexDefs)

				checkCurrStatesIndexes = false
			} else { // It's a node op.
				nodeName := name
				log.Printf(" nodeOp: %s, nodeName: %s", op[0:1], nodeName)

				register := "wanted"
				if op[0:1] == "-" {
					register = "unknown"
				}
				if test.params["register"] != "" {
					register = test.params["register"]
				}
				if test.params[nodeName+".register"] != "" {
					register = test.params[nodeName+".register"]
				}

				if register == "unknown" {
					nodesToRemove = append(nodesToRemove, nodeName)

					// Delay actual unknown registration / removal
					// until after rebalance finishes.
					continue
				}

				waitUntilEmptyCfgEventsNodeDefsWanted()

				mgr, err := startNodeManager(nodeDir(nodeName),
					cfg, nodeName, register, test.params, server)
				if err != nil || mgr == nil {
					t.Errorf("expected no err, got: %#v", err)
				}
				if mgr0 == nil {
					mgr0 = mgr
				}

				mgrs[nodeName] = mgr

				mgr.Kick("kick")

				waitUntilEmptyCfgEventsNodeDefsWanted()

				checkCurrStatesIndexes = true
			}
		}

		r, err := StartRebalance(cbgt.VERSION, cfg, ".",
			nodesToRemove,
			RebalanceOptions{
				HttpGet: httpGet,
			},
		)
		if (test.expStartErr && err == nil) ||
			(!test.expStartErr && err != nil) {
			t.Errorf("testi: %d, label: %q,"+
				" expStartErr: %v, but got: %v",
				testi, test.label,
				test.expStartErr, err)
		}

		if err != nil || r == nil {
			continue
		}

		progressCh := r.ProgressCh()
		if progressCh == nil {
			t.Errorf("expected progressCh")
		}

		err = nil
		for progress := range progressCh {
			if progress.Error != nil {
				err = progress.Error

				log.Printf("saw progress error: %#v\n", progress)
			}
		}

		r.Stop()

		if err != nil {
			t.Errorf("expected no end err, got: %v", err)
		}

		for _, nodeToRemove := range nodesToRemove {
			if mgrs[nodeToRemove] != nil {
				mgrs[nodeToRemove].Stop()
				delete(mgrs, nodeToRemove)
			}

			// TODO: Perhaps one day, the MCP will unregister the node;
			// for now, we unregister it "manually".
			if true {
				waitUntilEmptyCfgEventsNodeDefsWanted()

				mgr, err := startNodeManager(nodeDir(nodeToRemove),
					cfg, nodeToRemove, "unknown", test.params, server)
				if err != nil || mgr == nil {
					t.Errorf("expected no err, got: %#v", err)
				}

				mgr.Kick("kick")

				waitUntilEmptyCfgEventsNodeDefsWanted()

				mgr.Stop()
			}
		}

		endIndexDefs, endNodeDefs, endPlanPIndexes, endPlanPIndexesCAS, err :=
			cbgt.PlannerGetPlan(cfg, cbgt.VERSION, "")
		if err != nil ||
			endIndexDefs == nil ||
			endNodeDefs == nil ||
			endPlanPIndexes == nil ||
			endPlanPIndexesCAS == 0 {
			t.Errorf("expected no err, got: %#v", err)
		}

		expNodes := strings.Split(test.expNodes, " ")
		if len(expNodes) != len(endNodeDefs.NodeDefs) {
			t.Errorf("len(expNodes) != len(endNodeDefs.NodeDefs), "+
				" expNodes: %#v, endNodeDefs.NodeDefs: %#v",
				expNodes, endNodeDefs.NodeDefs)
		}

		for _, expNode := range expNodes {
			if endNodeDefs.NodeDefs[expNode] == nil {
				t.Errorf("didn't find expNode: %s,"+
					" expNodes: %#v, endNodeDefs.NodeDefs: %#v",
					expNode, expNodes, endNodeDefs.NodeDefs)
			}
		}

		expIndexes := strings.Split(test.expIndexes, " ")

		r.Visit(func(
			currStates CurrStates,
			currSeqs CurrSeqs,
			wantSeqs WantSeqs,
			nextMoves map[string]*blance.NextMoves) {
			if !checkCurrStatesIndexes {
				return
			}

			if len(currStates) != len(expIndexes) {
				t.Errorf("test.label: %s,"+
					" len(expIndexes) != len(currStates), "+
					" expIndexes: %#v, currStates: %#v, endIndexDefs: %#v",
					test.label, expIndexes, currStates, endIndexDefs)
			}
		})
	}
}
Esempio n. 5
0
func TestHandlersForEmptyManager(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	meh := &TestMEH{}
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
	err := mgr.Start("wanted")
	if err != nil {
		t.Errorf("expected start ok")
	}
	mgr.Kick("test-start-kick")

	mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
	mr.Write([]byte("hello"))
	mr.Write([]byte("world"))

	mgr.AddEvent([]byte(`"fizz"`))
	mgr.AddEvent([]byte(`"buzz"`))

	router, _, err := NewRESTRouter("v0", mgr, "static", "", mr,
		AssetDir, Asset)
	if err != nil || router == nil {
		t.Errorf("no mux router")
	}

	tests := []*RESTHandlerTest{
		{
			Desc:         "log on empty msg ring",
			Path:         "/api/log",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       http.StatusOK,
			ResponseBody: []byte(`{"messages":["hello","world"],"events":["fizz","buzz"]}`),
		},
		{
			Desc:   "cfg on empty manaager",
			Path:   "/api/cfg",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`"status":"ok"`:       true,
				`"indexDefs":null`:    true,
				`"nodeDefsKnown":{`:   true,
				`"nodeDefsWanted":{`:  true,
				`"planPIndexes":null`: true,
			},
		},
		{
			Desc:   "cfg refresh on empty, unchanged manager",
			Path:   "/api/cfgRefresh",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok"}`: true,
			},
		},
		{
			Desc:   "manager kick on empty, unchanged manager",
			Path:   "/api/managerKick",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok"}`: true,
			},
		},
		{
			Desc:   "manager meta",
			Path:   "/api/managerMeta",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`"status":"ok"`:    true,
				`"startSamples":{`: true,
			},
		},
		{
			Desc:   "feed stats when no feeds",
			Path:   "/api/stats",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{`: true,
				`}`: true,
			},
		},
		{
			Desc:         "list empty indexes",
			Path:         "/api/index",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       http.StatusOK,
			ResponseBody: []byte(`{"status":"ok","indexDefs":null}`),
		},
		{
			Desc:         "try to get a nonexistent index",
			Path:         "/api/index/NOT-AN-INDEX",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       400,
			ResponseBody: []byte(`index not found`),
		},
		{
			Desc:   "try to create a default index with no params",
			Path:   "/api/index/index-on-a-bad-server",
			Method: "PUT",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`rest_create_index: indexType is required`: true,
			},
		},
		{
			Desc:   "try to create a default index with no sourceType",
			Path:   "/api/index/index-on-a-bad-server",
			Method: "PUT",
			Params: url.Values{
				"indexType": []string{"no-care"},
			},
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`rest_create_index: sourceType is required`: true,
			},
		},
		{
			Desc:   "try to create a default index with bad indexType",
			Path:   "/api/index/index-on-a-bad-server",
			Method: "PUT",
			Params: url.Values{
				"indexType":  []string{"no-care"},
				"sourceType": []string{"couchbase"},
			},
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`unknown indexType`: true,
			},
		},
		{
			Desc:   "try to delete a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX",
			Method: "DELETE",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`no indexes`: true,
			},
		},
		{
			Desc:   "try to count a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX/count",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`could not get indexDefs`: true,
			},
		},
		{
			Desc:   "try to query a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX/query",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`could not get indexDefs`: true,
			},
		},
		{
			Desc:   "create an index with bogus indexType",
			Path:   "/api/index/idxBogusIndexType",
			Method: "PUT",
			Params: url.Values{
				"indexType":  []string{"not-a-real-index-type"},
				"sourceType": []string{"nil"},
			},
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`error`: true,
			},
		},
		{
			Desc:   "create a blackhole index",
			Path:   "/api/index/bh0",
			Method: "PUT",
			Params: url.Values{
				"indexType":  []string{"blackhole"},
				"sourceType": []string{"nil"},
			},
			Body:   nil,
			Status: 200,
		},
		{
			Desc:   "create a blackhole index via body",
			Path:   "/api/index/bh1",
			Method: "PUT",
			Body:   []byte(`{"type":"blackhole","sourceType":"nil"}`),
			Status: 200,
		},
		{
			Desc:   "create a blackhole index, bad params",
			Path:   "/api/index/bh2",
			Method: "PUT",
			Body:   []byte(`{"type":"blackhole","params":666,"sourceType":"nil"}`),
			Status: 400,
			ResponseMatch: map[string]bool{
				`cannot unmarshal number into Go value`: true,
			},
		},
		{
			Desc:   "create a blackhole index, bad params",
			Path:   "/api/index/bh2",
			Method: "PUT",
			Body:   []byte(`{"type":"blackhole","params":"hi","sourceType":"nil"}`),
			Status: 200,
		},
		{
			Desc:   "create a blackhole index, bad params",
			Path:   "/api/index/bh3",
			Method: "PUT",
			Body:   []byte(`{"type":"blackhole","params":{},"sourceType":"nil"}`),
			Status: 200,
		},
		{
			Desc:   "create a blackhole index, bad sourceParams",
			Path:   "/api/index/bh2s",
			Method: "PUT",
			Body:   []byte(`{"type":"blackhole","sourceParams":666,"sourceType":"nil"}`),
			Status: 400,
			ResponseMatch: map[string]bool{
				`cannot unmarshal number into Go value`: true,
			},
		},
		{
			Desc:   "create a blackhole index, bad sourceParams",
			Path:   "/api/index/bh2s",
			Method: "PUT",
			Body:   []byte(`{"type":"blackhole","sourceParams":"hi","sourceType":"nil"}`),
			Status: 200,
		},
		{
			Desc:   "create a blackhole index, bad sourceParams",
			Path:   "/api/index/bh3s",
			Method: "PUT",
			Body:   []byte(`{"type":"blackhole","sourceParams":{},"sourceType":"nil"}`),
			Status: 200,
		},
	}

	testRESTHandlers(t, tests, router)
}
Esempio n. 6
0
func TestHandlersForRuntimeOps(t *testing.T) {
	emptyDir, err := ioutil.TempDir("./tmp", "test")
	if err != nil {
		t.Errorf("tempdir err: %v", err)
	}
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	meh := &TestMEH{}
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
	err = mgr.Start("wanted")
	if err != nil {
		t.Errorf("expected no start err, got: %v", err)
	}

	mgr.Kick("test-start-kick")

	mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
	mr.Write([]byte("hello"))
	mr.Write([]byte("world"))

	router, _, err := NewRESTRouter("v0", mgr, "static", "", mr,
		AssetDir, Asset)
	if err != nil || router == nil {
		t.Errorf("no mux router")
	}

	tests := []*RESTHandlerTest{
		{
			Path:   "/api/runtime",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				"arch":       true,
				"go":         true,
				"GOMAXPROCS": true,
				"GOROOT":     true,
			},
		},
		{
			Path:          "/api/runtime/args",
			Method:        "GET",
			Params:        nil,
			Body:          nil,
			Status:        http.StatusOK,
			ResponseMatch: map[string]bool{
			// Actual production args are different from "go test" context.
			},
		},
		{
			Path:         "/api/runtime/gc",
			Method:       "POST",
			Params:       nil,
			Body:         nil,
			Status:       http.StatusOK,
			ResponseBody: []byte(nil),
		},
		{
			Path:   "/api/runtime/statsMem",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				"Alloc":      true,
				"TotalAlloc": true,
			},
		},
		{
			Path:   "/api/runtime/profile/cpu",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				"incorrect or missing secs parameter": true,
			},
		},
	}

	testRESTHandlers(t, tests, router)
}
Esempio n. 7
0
func TestManagerIndexControl(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	m := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", nil)
	if err := m.Start("wanted"); err != nil {
		t.Errorf("expected Manager.Start() to work, err: %v", err)
	}
	sourceParams := ""
	if err := m.CreateIndex("primary", "default", "123", sourceParams,
		"bleve", "foo", "", cbgt.PlanParams{}, ""); err != nil {
		t.Errorf("expected CreateIndex() to work, err: %v", err)
	}
	m.Kick("test0")
	m.PlannerNOOP("test0")

	err := m.IndexControl("foo", "wrong-uuid", "", "", "")
	if err == nil {
		t.Errorf("expected err on wrong UUID")
	}

	indexDefs, _, _ := cbgt.CfgGetIndexDefs(cfg)
	npp := indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
	if npp != nil {
		t.Errorf("expected nil npp")
	}

	err = m.IndexControl("foo", "", "", "", "")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
	if npp == nil {
		t.Errorf("expected npp")
	}
	if npp[""] != nil {
		t.Errorf("expected nil npp.sub")
	}

	err = m.IndexControl("foo", "", "disallow", "", "")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
	if npp == nil {
		t.Errorf("expected npp")
	}
	if npp[""] == nil {
		t.Errorf("expected npp.sub")
	}
	if npp[""].CanRead {
		t.Errorf("expected CanRead false")
	}
	if !npp[""].CanWrite {
		t.Errorf("expected CanWrite")
	}

	err = m.IndexControl("foo", "", "", "", "")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
	if npp == nil {
		t.Errorf("expected npp")
	}
	if npp[""] == nil {
		t.Errorf("expected npp.sub")
	}
	if npp[""].CanRead {
		t.Errorf("expected CanRead false")
	}
	if !npp[""].CanWrite {
		t.Errorf("expected CanWrite")
	}

	err = m.IndexControl("foo", "", "", "pause", "")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
	if npp == nil {
		t.Errorf("expected npp")
	}
	if npp[""] == nil {
		t.Errorf("expected npp.sub")
	}
	if npp[""].CanRead {
		t.Errorf("expected CanRead false")
	}
	if npp[""].CanWrite {
		t.Errorf("expected CanWrite false")
	}

	err = m.IndexControl("foo", "", "", "", "")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
	if npp == nil {
		t.Errorf("expected npp")
	}
	if npp[""] == nil {
		t.Errorf("expected npp.sub")
	}
	if npp[""].CanRead {
		t.Errorf("expected CanRead false")
	}
	if npp[""].CanWrite {
		t.Errorf("expected CanWrite false")
	}

	err = m.IndexControl("foo", "", "", "resume", "")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
	if npp == nil {
		t.Errorf("expected npp")
	}
	if npp[""] == nil {
		t.Errorf("expected npp.sub")
	}
	if npp[""].CanRead {
		t.Errorf("expected CanRead false")
	}
	if !npp[""].CanWrite {
		t.Errorf("expected CanWrite")
	}

	err = m.IndexControl("foo", "", "allow", "resume", "")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	npp = indexDefs.IndexDefs["foo"].PlanParams.NodePlanParams[""]
	if npp == nil {
		t.Errorf("expected npp")
	}
	if npp[""] != nil {
		t.Errorf("expected nil npp.sub")
	}

	if indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen {
		t.Errorf("expected not yet frozen")
	}
	err = m.IndexControl("foo", "", "", "", "freeze")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	if !indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen {
		t.Errorf("expected frozen")
	}

	err = m.IndexControl("foo", "", "", "", "unfreeze")
	if err != nil {
		t.Errorf("expected ok")
	}
	indexDefs, _, _ = cbgt.CfgGetIndexDefs(cfg)
	if indexDefs.IndexDefs["foo"].PlanParams.PlanFrozen {
		t.Errorf("expected not frozen")
	}
}
Esempio n. 8
0
func TestPreparePerms(t *testing.T) {

	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	meh := &TestMEH{}
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
	mgr.Start("wanted")
	mgr.Kick("test-start-kick")

	mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)

	router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
	if err != nil || router == nil {
		t.Errorf("no mux router")
	}
	router.KeepContext = true // so we can see the mux vars

	s := &stubDefinitionLookuper{
		pindexes: testPIndexesByName,
		defs: &cbgt.IndexDefs{
			IndexDefs: testIndexDefsByName,
		},
	}

	tests := []struct {
		method string
		uri    string
		body   []byte
		path   string
		perms  []string
		err    error
	}{
		// case with valid perm not containing source
		{
			method: http.MethodGet,
			uri:    "/api/index",
			path:   "/api/index",
			perms:  []string{"cluster.bucket.fts!read"},
		},
		// case with valid index name
		{
			method: http.MethodGet,
			uri:    "/api/index/i1",
			path:   "/api/index/{indexName}",
			perms:  []string{"cluster.bucket[s1].fts!read"},
		},
		// case with invalid index name
		{
			method: http.MethodGet,
			uri:    "/api/index/x1",
			path:   "/api/index/{indexName}",
			err:    errIndexNotFound,
		},
		// case with invalid index name (actuall pindex name)
		{
			method: http.MethodGet,
			uri:    "/api/index/p1",
			path:   "/api/index/{indexName}",
			err:    errIndexNotFound,
		},
		// case with valid pindex name
		{
			method: http.MethodGet,
			uri:    "/api/pindex/p1",
			path:   "/api/pindex/{pindexName}",
			perms:  []string{"cluster.bucket[s3].fts!read"},
		},
		// case with invalid pindex name
		{
			method: http.MethodGet,
			uri:    "/api/pindex/y1",
			path:   "/api/pindex/{pindexName}",
			err:    errPIndexNotFound,
		},
		// case with invalid pindex name (actually index name)
		{
			method: http.MethodGet,
			uri:    "/api/pindex/i1",
			path:   "/api/pindex/{pindexName}",
			err:    errPIndexNotFound,
		},
		// case with valid alias, but this operation does NOT expand alias
		{
			method: http.MethodGet,
			uri:    "/api/index/a1",
			path:   "/api/index/{indexName}",
			perms:  []string{"cluster.bucket.fts!read"},
		},
		// case with valid alias, and this operation DOES expand alias
		{
			method: http.MethodGet,
			uri:    "/api/index/a1/count",
			path:   "/api/index/{indexName}/count",
			perms:  []string{"cluster.bucket[s1].fts!read"},
		},
		// case with valid alias (multi), and this operation DOES expand alias
		{
			method: http.MethodGet,
			uri:    "/api/index/a2/count",
			path:   "/api/index/{indexName}/count",
			perms:  []string{"cluster.bucket[s1].fts!read", "cluster.bucket[s2].fts!read"},
		},
		// case with valid alias (containing another alias),
		// and this operation DOES expand alias
		{
			method: http.MethodGet,
			uri:    "/api/index/a3/count",
			path:   "/api/index/{indexName}/count",
			perms:  []string{"cluster.bucket[s1].fts!read", "cluster.bucket[s2].fts!read"},
		},
		// test special case for creating new index
		{
			method: http.MethodPut,
			uri:    "/api/index/anewone",
			body:   []byte(`{"type":"fulltext-index","sourceType":"couchbase","sourceName":"abucket"}`),
			path:   "/api/index/{indexName}",
			perms:  []string{"cluster.bucket[abucket].fts!write"},
		},
	}

	for i, test := range tests {
		var r io.Reader
		if test.body != nil {
			r = bytes.NewBuffer(test.body)
		}
		req, err := http.NewRequest(test.method, test.uri, r)
		if err != nil {
			t.Fatal(err)
		}

		// this actually executes things, which will usually fail
		// which is unrelated to what we're testing
		// but its the best i could do
		record := httptest.NewRecorder()
		router.ServeHTTP(record, req)

		// set the request body again, as its been consumed :(
		if test.body != nil {
			req.Body = ioutil.NopCloser(bytes.NewBuffer(test.body))
		}

		actualPerms, err := preparePerms(s, req, test.method, test.path)
		if err != test.err {
			t.Errorf("test %d, expected err %v, got %v", i, test.err, err)
		}
		sort.Strings(actualPerms)
		if !reflect.DeepEqual(actualPerms, test.perms) {
			t.Errorf("test %d, expected %v, got %v", i, test.perms, actualPerms)
		}
	}

}
Esempio n. 9
0
func TestSourceNamesFromReq(t *testing.T) {

	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	meh := &TestMEH{}
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
	mgr.Start("wanted")
	mgr.Kick("test-start-kick")

	mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)

	router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
	if err != nil || router == nil {
		t.Errorf("no mux router")
	}
	router.KeepContext = true // so we can see the mux vars

	s := &stubDefinitionLookuper{
		pindexes: testPIndexesByName,
		defs: &cbgt.IndexDefs{
			IndexDefs: testIndexDefsByName,
		},
	}

	tests := []struct {
		method  string
		uri     string
		path    string
		sources []string
		err     error
	}{
		// case with valid index name
		{
			method:  http.MethodGet,
			uri:     "/api/index/i1",
			path:    "/api/index/{indexName}",
			sources: []string{"s1"},
		},
		// case with invalid index name
		{
			method: http.MethodGet,
			uri:    "/api/index/x1",
			path:   "/api/index/{indexName}",
			err:    errIndexNotFound,
		},
		// case with invalid index name (actuall pindex name)
		{
			method: http.MethodGet,
			uri:    "/api/index/p1",
			path:   "/api/index/{indexName}",
			err:    errIndexNotFound,
		},
		// case with valid pindex name
		{
			method:  http.MethodGet,
			uri:     "/api/pindex/p1",
			path:    "/api/pindex/{pindexName}",
			sources: []string{"s3"},
		},
		// case with invalid pindex name
		{
			method: http.MethodGet,
			uri:    "/api/pindex/y1",
			path:   "/api/pindex/{pindexName}",
			err:    errPIndexNotFound,
		},
		// case with invalid pindex name (actually index name)
		{
			method: http.MethodGet,
			uri:    "/api/pindex/i1",
			path:   "/api/pindex/{pindexName}",
			err:    errPIndexNotFound,
		},
		// case with valid alias, but this operation does NOT expand alias
		{
			method:  http.MethodGet,
			uri:     "/api/index/a1",
			path:    "/api/index/{indexName}",
			sources: []string{},
		},
		// case with valid alias, and this operation DOES expand alias
		{
			method:  http.MethodGet,
			uri:     "/api/index/a1/count",
			path:    "/api/index/{indexName}/count",
			sources: []string{"s1"},
		},
		// case with valid alias (multi), and this operation DOES expand alias
		{
			method:  http.MethodGet,
			uri:     "/api/index/a2/count",
			path:    "/api/index/{indexName}/count",
			sources: []string{"s1", "s2"},
		},
		// case with valid alias (containing another alias),
		// and this operation DOES expand alias
		{
			method:  http.MethodGet,
			uri:     "/api/index/a3/count",
			path:    "/api/index/{indexName}/count",
			sources: []string{"s1", "s2"},
		},
	}

	for i, test := range tests {
		req, err := http.NewRequest(test.method, test.uri, nil)
		if err != nil {
			t.Fatal(err)
		}

		// this actually executes things, which will usually fail
		// which is unrelated to what we're testing
		// but its the best i could do
		record := httptest.NewRecorder()
		router.ServeHTTP(record, req)

		actualNames, err := sourceNamesFromReq(s, req, test.method, test.path)
		if err != test.err {
			t.Errorf("test %d, expected err %v, got %v", i, test.err, err)
		}
		sort.Strings(actualNames)
		if !reflect.DeepEqual(actualNames, test.sources) {
			t.Errorf("test %d, expected %v, got %v", i, test.sources, actualNames)
		}
	}
}
Esempio n. 10
0
func testHandlersWithOnePartitionPrimaryFeedPartialRollbackMoss(t *testing.T,
	rollbackToSeq uint64,
	rollbackDesc string,
	afterRollbackCountResponseMatch map[string]bool) {
	BlevePIndexAllowMossPrev := BlevePIndexAllowMoss
	BlevePIndexAllowMoss = true

	bleveConfigDefaultKVStorePrev := bleve.Config.DefaultKVStore
	bleve.Config.DefaultKVStore = "mossStore"

	bleveMossRegistryCollectionOptionsFTSPrev := bleveMoss.RegistryCollectionOptions["fts"]
	bleveMoss.RegistryCollectionOptions["fts"] = moss.CollectionOptions{}

	defer func() {
		BlevePIndexAllowMoss = BlevePIndexAllowMossPrev
		bleve.Config.DefaultKVStore = bleveConfigDefaultKVStorePrev
		bleveMoss.RegistryCollectionOptions["fts"] = bleveMossRegistryCollectionOptionsFTSPrev
	}()

	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	meh := &TestMEH{}
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
	mgr.Start("wanted")
	mgr.Kick("test-start-kick")

	mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)

	router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
	if err != nil || router == nil {
		t.Errorf("no mux router")
	}

	var pindex *cbgt.PIndex
	var feed *cbgt.PrimaryFeed

	waitForPersistence := func() {
		for i := 0; i < 100; i++ {
			stats := map[string]interface{}{
				"num_recs_to_persist": float64(0),
			}
			err := addPIndexStats(pindex, stats)
			if err != nil {
				t.Errorf("expected nil addPIndexStats err, got: %v", err)
			}
			v, ok := stats["num_recs_to_persist"]
			if ok {
				nrtp, ok := v.(float64)
				if ok && nrtp <= 0.0 {
					return
				}
			}
			time.Sleep(50 * time.Millisecond)
		}
		t.Errorf("persistence took too long!")
	}

	tests := []*RESTHandlerTest{
		{
			Desc:   "create dest feed with 1 partition for rollback",
			Path:   "/api/index/idx0",
			Method: "PUT",
			Params: url.Values{
				"indexType": []string{"fulltext-index"},
				"indexParams": []string{
					`{"store":{"mossStoreOptions":{"CompactionPercentage":100000.0}}}`,
				}, // Never compact during this test.
				"sourceType":   []string{"primary"},
				"sourceParams": []string{`{"numPartitions":1}`},
			},
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok"}`: true,
			},
			After: func() {
				time.Sleep(10 * time.Millisecond)
				mgr.Kick("after-rollback")
				feeds, pindexes := mgr.CurrentMaps()
				if len(feeds) != 1 {
					t.Errorf("expected to be 1 feed, got feeds: %+v", feeds)
				}
				if len(pindexes) != 1 {
					t.Errorf("expected to be 1 pindex,"+
						" got pindexes: %+v", pindexes)
				}
				for _, f := range feeds {
					var ok bool
					feed, ok = f.(*cbgt.PrimaryFeed)
					if !ok {
						t.Errorf("expected the 1 feed to be a PrimaryFeed")
					}
				}
				for _, p := range pindexes {
					pindex = p
				}
			},
		},
		{
			Before: func() {
				partition := "0"
				snapStart := uint64(1)
				snapEnd := uint64(10)
				err = feed.SnapshotStart(partition, snapStart, snapEnd)
				if err != nil {
					t.Errorf("expected no err on snapshot-start")
				}
			},
			Desc:   "count idx0 0 when snapshot just started, pre-rollback",
			Path:   "/api/index/idx0/count",
			Method: "GET",
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok","count":0}`: true,
			},
		},
		{
			Before: func() {
				partition := "0"
				key := []byte("ss1-k1")
				seq := uint64(1)
				val := []byte(`{"foo":"bar","yow":"wow"}`)
				err = feed.DataUpdate(partition, key, seq, val,
					0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
				if err != nil {
					t.Errorf("expected no err on data-update")
				}
			},
			Desc: "count idx0 should be 0 when got an update (doc created)" +
				" but snapshot not ended, pre-rollback",
			Path:   "/api/index/idx0/count",
			Method: "GET",
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok","count":0}`: true,
			},
		},
		{
			Before: func() {
				partition := "0"
				key := []byte("ss1-k2")
				seq := uint64(2)
				val := []byte(`{"foo":"bing","yow":"wow"}`)
				err = feed.DataUpdate(partition, key, seq, val,
					0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
				if err != nil {
					t.Errorf("expected no err on data-update")
				}
			},
			Desc: "count idx0 0 when got 2nd update (another doc created)" +
				" but snapshot not ended, pre-rollback",
			Path:   "/api/index/idx0/count",
			Method: "GET",
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok","count":0}`: true,
			},
		},
		{
			Before: func() {
				partition := "0"
				snapStart := uint64(11)
				snapEnd := uint64(20)
				err = feed.SnapshotStart(partition, snapStart, snapEnd)
				if err != nil {
					t.Errorf("expected no err on snapshot-start")
				}
				waitForPersistence()
			},
			Desc:   "count idx0 2 when 1st snapshot ended, pre-rollback",
			Path:   "/api/index/idx0/count",
			Method: "GET",
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok","count":2}`: true,
			},
		},
		{
			Before: func() {
				partition := "0"
				key := []byte("ss11-k11")
				seq := uint64(11)
				val := []byte(`{"foo":"baz","yow":"NOPE"}`)
				err = feed.DataUpdate(partition, key, seq, val,
					0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
				if err != nil {
					t.Errorf("expected no err on data-update")
				}
			},
			Desc: "count idx0 should be 2 when got an update (doc created)" +
				" in 2nd snapshot, 2nd snapshot not ended, pre-rollback",
			Path:   "/api/index/idx0/count",
			Method: "GET",
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok","count":2}`: true,
			},
		},
		{
			Before: func() {
				partition := "0"
				snapStart := uint64(21)
				snapEnd := uint64(30)
				err = feed.SnapshotStart(partition, snapStart, snapEnd)
				if err != nil {
					t.Errorf("expected no err on snapshot-start")
				}
				waitForPersistence()
			},
			Desc:   "count idx0 3 when 2nd snapshot ended, pre-rollback",
			Path:   "/api/index/idx0/count",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok","count":3}`: true,
			},
		},
		{
			Before: func() {
				partition := "0"
				key := []byte("ss21-k21")
				seq := uint64(21)
				val := []byte(`{"foo":"baz","yow":"whoa"}`)
				err = feed.DataUpdate(partition, key, seq, val,
					0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
				if err != nil {
					t.Errorf("expected no err on data-update")
				}
			},
			Desc: "count idx0 should be 3 when got an update (doc created)" +
				" in 3rd snapshot, 3rd snapshot not ended, pre-rollback",
			Path:   "/api/index/idx0/count",
			Method: "GET",
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok","count":3}`: true,
			},
		},
		{
			Before: func() {
				partition := "0"
				snapStart := uint64(31)
				snapEnd := uint64(40)
				err = feed.SnapshotStart(partition, snapStart, snapEnd)
				if err != nil {
					t.Errorf("expected no err on snapshot-start")
				}
				waitForPersistence()
			},
			Desc:   "count idx0 4 when 3rd snapshot ended, pre-rollback",
			Path:   "/api/index/idx0/count",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok","count":4}`: true,
			},
		},
		{
			// Rollback to a high seq number, so it's a no-op
			// w.r.t. doing an actual rollback.
			Before: func() {
				partition := "0"
				err = feed.Rollback(partition, rollbackToSeq)
				if err != nil {
					t.Errorf("expected no err on rollback, got: %v", err)
				}
				// NOTE: We might check right after rollback but before
				// we get a kick, but unfortunately results will be race-y.
				time.Sleep(100 * time.Millisecond)
				mgr.Kick("after-rollback")
			},
			Desc:          rollbackDesc,
			Path:          "/api/index/idx0/count",
			Method:        "GET",
			Params:        nil,
			Body:          nil,
			Status:        http.StatusOK,
			ResponseMatch: afterRollbackCountResponseMatch,
		},
	}

	testRESTHandlers(t, tests, router)
}
Esempio n. 11
0
// Emits markdown docs of cbft's REST API.
func main() {
	rand.Seed(0)

	dataDir, _ := ioutil.TempDir("./tmp", "data")
	defer os.RemoveAll(dataDir)

	cfg := cbgt.NewCfgMem()
	tags := []string(nil)
	container := ""
	weight := 1
	extras := ""
	bindHttp := "0.0.0.0:8095"

	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		tags, container, weight, extras, bindHttp,
		dataDir, "http://localhost:8091", nil)

	sourceType := "nil"
	sourceName := ""
	sourceUUID := ""
	sourceParams := ""
	indexType := "blackhole"
	indexName := "myFirstIndex"
	indexParams := ""
	prevIndexUUID := ""

	mgr.Start("wanted")

	err := mgr.CreateIndex(
		sourceType, sourceName, sourceUUID, sourceParams,
		indexType, indexName, indexParams,
		cbgt.PlanParams{}, prevIndexUUID)
	if err != nil {
		log.Fatalf("could not create myFirstIndex, err: %v", err)
	}

	staticDir := ""
	staticETag := ""

	mr, _ := cbgt.NewMsgRing(ioutil.Discard, 1)

	router, meta, err :=
		cbft.NewRESTRouter(VERSION, mgr, staticDir, staticETag, mr)
	if err != nil {
		log.Panic(err)
	}

	mainCategoriesMap := map[string]bool{}
	mainCategories := []string(nil)

	subCategoriesMap := map[string]bool{}
	subCategories := []string(nil)

	paths := []string(nil)
	for path := range meta {
		paths = append(paths, path)

		m := meta[path]
		if m.Opts != nil {
			category := m.Opts["_category"]

			mainCategory, _, subCategory, _ := categoryParse(category)

			if !mainCategoriesMap[mainCategory] {
				mainCategoriesMap[mainCategory] = true
				mainCategories = append(mainCategories, mainCategory)
			}

			if !subCategoriesMap[subCategory] {
				subCategoriesMap[subCategory] = true
				subCategories = append(subCategories, subCategory)
			}
		}
	}

	sort.Strings(mainCategories)
	sort.Strings(subCategories)
	sort.Strings(paths)

	fmt.Printf("# API Reference\n\n")

	for _, mainCategory := range mainCategories {
		mainCategoryFirst := true

		for _, subCategory := range subCategories {
			subCategoryFirst := true

			for _, path := range paths {
				m := meta[path]

				category := ""
				if m.Opts != nil {
					category = m.Opts["_category"]
				}

				mc, mcVis, sc, scVis := categoryParse(category)
				if mc != mainCategory ||
					sc != subCategory {
					continue
				}

				if m.Opts != nil &&
					m.Opts["_status"] == "private" {
					continue
				}

				if mainCategoryFirst {
					fmt.Printf("---\n\n# %s\n\n", mcVis)
				}
				mainCategoryFirst = false

				if subCategoryFirst {
					fmt.Printf("---\n\n## %s\n\n", scVis)
				}
				subCategoryFirst = false

				pathParts := strings.Split(path, " ")
				fmt.Printf("---\n\n")
				fmt.Printf("%s `%s`\n\n", m.Method, pathParts[0])

				if m.Opts != nil && m.Opts["_about"] != "" {
					fmt.Printf("%s\n\n", m.Opts["_about"])
				}

				optNames := []string(nil)
				for optName := range m.Opts {
					if optName != "" && !strings.HasPrefix(optName, "_") {
						optNames = append(optNames, optName)
					}
				}
				sort.Strings(optNames)

				for _, optName := range optNames {
					fmt.Printf("**%s**: %s\n\n", optName, m.Opts[optName])
				}

				if m.Opts != nil && m.Opts[""] != "" {
					fmt.Printf("%s\n\n", m.Opts[""])
				}

				if m.Method == "GET" && !skipSampleResponses[pathParts[0]] {
					url := strings.Replace(pathParts[0],
						"{indexName}", "myFirstIndex", 1)

					res := sampleRequest(router, m.Method, url, nil)
					if res != nil {
						var j map[string]interface{}

						err = json.Unmarshal(res, &j)
						if err == nil {
							s, err := json.MarshalIndent(j, "    ", "  ")
							if err == nil {
								fmt.Printf("Sample response:\n\n    %s\n\n", s)
							}
						}
					}
				}
			}
		}
	}

	fmt.Printf("---\n\nCopyright (c) 2015 Couchbase, Inc.\n")
}