func TestNewRESTRouter(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
ring, err := cbgt.NewMsgRing(nil, 1)
cfg := cbgt.NewCfgMem()
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000",
emptyDir, "some-datasource", nil)
r, meta, err := NewRESTRouter("v0", mgr, emptyDir, "", ring,
AssetDir, Asset)
if r == nil || meta == nil || err != nil {
t.Errorf("expected no errors")
}
mgr = cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
[]string{"queryer", "anotherTag"},
"", 1, "", ":1000", emptyDir, "some-datasource", nil)
r, meta, err = NewRESTRouter("v0", mgr, emptyDir, "", ring,
AssetDir, Asset)
if r == nil || meta == nil || err != nil {
t.Errorf("expected no errors")
}
}
func TestMainStart(t *testing.T) {
mr, err := cbgt.NewMsgRing(os.Stderr, 1000)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
router, err := MainStart(nil, cbgt.NewUUID(), nil, "", 1, "", ":1000",
"bad data dir", "./static", "etag", "", "", mr, "k=v,k2=v2")
if router != nil || err == nil {
t.Errorf("expected empty server string to fail mainStart()")
}
router, err = MainStart(nil, cbgt.NewUUID(), nil, "", 1, "", ":1000",
"bad data dir", "./static", "etag", "bad server", "", mr, "")
if router != nil || err == nil {
t.Errorf("expected bad server string to fail mainStart()")
}
}
func main() {
flag.Parse()
if flags.Help {
flag.Usage()
os.Exit(2)
}
if flags.Version {
fmt.Printf("%s main: %s, data: %s\n", path.Base(os.Args[0]),
VERSION, cbgt.VERSION)
os.Exit(0)
}
if os.Getenv("GOMAXPROCS") == "" {
runtime.GOMAXPROCS(runtime.NumCPU())
}
mr, err := cbgt.NewMsgRing(os.Stderr, 1000)
if err != nil {
log.Fatalf("main: could not create MsgRing, err: %v", err)
}
log.SetOutput(mr)
log.Printf("main: %s started (%s/%s)",
os.Args[0], VERSION, cbgt.VERSION)
rand.Seed(time.Now().UTC().UnixNano())
go cmd.DumpOnSignalForPlatform()
MainWelcome(flagAliases)
s, err := os.Stat(flags.DataDir)
if err != nil {
if os.IsNotExist(err) {
if flags.DataDir == DEFAULT_DATA_DIR {
log.Printf("main: creating data directory, dataDir: %s",
flags.DataDir)
err = os.Mkdir(flags.DataDir, 0700)
if err != nil {
log.Fatalf("main: could not create data directory,"+
" dataDir: %s, err: %v", flags.DataDir, err)
}
} else {
log.Fatalf("main: data directory does not exist,"+
" dataDir: %s", flags.DataDir)
return
}
} else {
log.Fatalf("main: could not access data directory,"+
" dataDir: %s, err: %v", flags.DataDir, err)
return
}
} else {
if !s.IsDir() {
log.Fatalf("main: not a directory, dataDir: %s", flags.DataDir)
return
}
}
wd, err := os.Getwd()
if err != nil {
log.Fatalf("main: os.Getwd, err: %#v", err)
return
}
log.Printf("main: curr dir: %q", wd)
dataDirAbs, err := filepath.Abs(flags.DataDir)
if err != nil {
log.Fatalf("main: filepath.Abs, err: %#v", err)
return
}
log.Printf("main: data dir: %q", dataDirAbs)
// If cfg is down, we error, leaving it to some user-supplied
// outside watchdog to backoff and restart/retry.
cfg, err := cmd.MainCfg(cmdName, flags.CfgConnect,
flags.BindHttp, flags.Register, flags.DataDir)
if err != nil {
if err == cmd.ErrorBindHttp {
log.Fatalf("%v", err)
return
}
log.Fatalf("main: could not start cfg, cfgConnect: %s, err: %v\n"+
" Please check that your -cfg/-cfgConnect parameter (%q)\n"+
" is correct and/or that your configuration provider\n"+
" is available.",
flags.CfgConnect, err, flags.CfgConnect)
return
}
uuid := flags.UUID
if uuid != "" {
uuidPath :=
flags.DataDir + string(os.PathSeparator) + cmdName + ".uuid"
err = ioutil.WriteFile(uuidPath, []byte(uuid), 0600)
if err != nil {
log.Fatalf("main: could not write uuidPath: %s\n"+
" Please check that your -data/-dataDir parameter (%q)\n"+
" is to a writable directory where %s can persist data.",
uuidPath, flags.DataDir, cmdName)
return
}
}
if uuid == "" {
uuid, err = cmd.MainUUID(cmdName, flags.DataDir)
if err != nil {
log.Fatalf(fmt.Sprintf("%v", err))
return
}
}
var tagsArr []string
if flags.Tags != "" {
tagsArr = strings.Split(flags.Tags, ",")
}
expvars.Set("indexes", bleveHttp.IndexStats())
router, err := MainStart(cfg, uuid, tagsArr,
flags.Container, flags.Weight, flags.Extra,
flags.BindHttp, flags.DataDir,
flags.StaticDir, flags.StaticETag,
flags.Server, flags.Register, mr, flags.Options)
if err != nil {
log.Fatal(err)
}
if flags.Register == "unknown" {
log.Printf("main: unregistered node; now exiting")
os.Exit(0)
}
http.Handle("/", router)
log.Printf("main: listening on: %s", flags.BindHttp)
u := flags.BindHttp
if u[0] == ':' {
u = "localhost" + u
}
if strings.HasPrefix(u, "0.0.0.0:") {
u = "localhost" + u[len("0.0.0.0"):]
}
log.Printf("------------------------------------------------------------")
log.Printf("web UI / REST API is available: http://%s", u)
log.Printf("------------------------------------------------------------")
err = http.ListenAndServe(flags.BindHttp, nil)
if err != nil {
log.Fatalf("main: listen, err: %v\n"+
" Please check that your -bindHttp parameter (%q)\n"+
" is correct and available.", err, flags.BindHttp)
}
}
func TestHandlersForEmptyManager(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
err := mgr.Start("wanted")
if err != nil {
t.Errorf("expected start ok")
}
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
mr.Write([]byte("hello"))
mr.Write([]byte("world"))
mgr.AddEvent([]byte(`"fizz"`))
mgr.AddEvent([]byte(`"buzz"`))
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr,
AssetDir, Asset)
if err != nil || router == nil {
t.Errorf("no mux router")
}
tests := []*RESTHandlerTest{
{
Desc: "log on empty msg ring",
Path: "/api/log",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseBody: []byte(`{"messages":["hello","world"],"events":["fizz","buzz"]}`),
},
{
Desc: "cfg on empty manaager",
Path: "/api/cfg",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`"status":"ok"`: true,
`"indexDefs":null`: true,
`"nodeDefsKnown":{`: true,
`"nodeDefsWanted":{`: true,
`"planPIndexes":null`: true,
},
},
{
Desc: "cfg refresh on empty, unchanged manager",
Path: "/api/cfgRefresh",
Method: "POST",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok"}`: true,
},
},
{
Desc: "manager kick on empty, unchanged manager",
Path: "/api/managerKick",
Method: "POST",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok"}`: true,
},
},
{
Desc: "manager meta",
Path: "/api/managerMeta",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`"status":"ok"`: true,
`"startSamples":{`: true,
},
},
{
Desc: "feed stats when no feeds",
Path: "/api/stats",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{`: true,
`}`: true,
},
},
{
Desc: "list empty indexes",
Path: "/api/index",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseBody: []byte(`{"status":"ok","indexDefs":null}`),
},
{
Desc: "try to get a nonexistent index",
Path: "/api/index/NOT-AN-INDEX",
Method: "GET",
Params: nil,
Body: nil,
Status: 400,
ResponseBody: []byte(`index not found`),
},
{
Desc: "try to create a default index with no params",
Path: "/api/index/index-on-a-bad-server",
Method: "PUT",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`rest_create_index: indexType is required`: true,
},
},
{
Desc: "try to create a default index with no sourceType",
Path: "/api/index/index-on-a-bad-server",
Method: "PUT",
Params: url.Values{
"indexType": []string{"no-care"},
},
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`rest_create_index: sourceType is required`: true,
},
},
{
Desc: "try to create a default index with bad indexType",
Path: "/api/index/index-on-a-bad-server",
Method: "PUT",
Params: url.Values{
"indexType": []string{"no-care"},
"sourceType": []string{"couchbase"},
},
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`unknown indexType`: true,
},
},
{
Desc: "try to delete a nonexistent index when no indexes",
Path: "/api/index/NOT-AN-INDEX",
Method: "DELETE",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`no indexes`: true,
},
},
{
Desc: "try to count a nonexistent index when no indexes",
Path: "/api/index/NOT-AN-INDEX/count",
Method: "GET",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`could not get indexDefs`: true,
},
},
{
Desc: "try to query a nonexistent index when no indexes",
Path: "/api/index/NOT-AN-INDEX/query",
Method: "POST",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`could not get indexDefs`: true,
},
},
{
Desc: "create an index with bogus indexType",
Path: "/api/index/idxBogusIndexType",
Method: "PUT",
Params: url.Values{
"indexType": []string{"not-a-real-index-type"},
"sourceType": []string{"nil"},
},
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
`error`: true,
},
},
{
Desc: "create a blackhole index",
Path: "/api/index/bh0",
Method: "PUT",
Params: url.Values{
"indexType": []string{"blackhole"},
"sourceType": []string{"nil"},
},
Body: nil,
Status: 200,
},
{
Desc: "create a blackhole index via body",
Path: "/api/index/bh1",
Method: "PUT",
Body: []byte(`{"type":"blackhole","sourceType":"nil"}`),
Status: 200,
},
{
Desc: "create a blackhole index, bad params",
Path: "/api/index/bh2",
Method: "PUT",
Body: []byte(`{"type":"blackhole","params":666,"sourceType":"nil"}`),
Status: 400,
ResponseMatch: map[string]bool{
`cannot unmarshal number into Go value`: true,
},
},
{
Desc: "create a blackhole index, bad params",
Path: "/api/index/bh2",
Method: "PUT",
Body: []byte(`{"type":"blackhole","params":"hi","sourceType":"nil"}`),
Status: 200,
},
{
Desc: "create a blackhole index, bad params",
Path: "/api/index/bh3",
Method: "PUT",
Body: []byte(`{"type":"blackhole","params":{},"sourceType":"nil"}`),
Status: 200,
},
{
Desc: "create a blackhole index, bad sourceParams",
Path: "/api/index/bh2s",
Method: "PUT",
Body: []byte(`{"type":"blackhole","sourceParams":666,"sourceType":"nil"}`),
Status: 400,
ResponseMatch: map[string]bool{
`cannot unmarshal number into Go value`: true,
},
},
{
Desc: "create a blackhole index, bad sourceParams",
Path: "/api/index/bh2s",
Method: "PUT",
Body: []byte(`{"type":"blackhole","sourceParams":"hi","sourceType":"nil"}`),
Status: 200,
},
{
Desc: "create a blackhole index, bad sourceParams",
Path: "/api/index/bh3s",
Method: "PUT",
Body: []byte(`{"type":"blackhole","sourceParams":{},"sourceType":"nil"}`),
Status: 200,
},
}
testRESTHandlers(t, tests, router)
}
func TestPreparePerms(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
mgr.Start("wanted")
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
if err != nil || router == nil {
t.Errorf("no mux router")
}
router.KeepContext = true // so we can see the mux vars
s := &stubDefinitionLookuper{
pindexes: testPIndexesByName,
defs: &cbgt.IndexDefs{
IndexDefs: testIndexDefsByName,
},
}
tests := []struct {
method string
uri string
body []byte
path string
perms []string
err error
}{
// case with valid perm not containing source
{
method: http.MethodGet,
uri: "/api/index",
path: "/api/index",
perms: []string{"cluster.bucket.fts!read"},
},
// case with valid index name
{
method: http.MethodGet,
uri: "/api/index/i1",
path: "/api/index/{indexName}",
perms: []string{"cluster.bucket[s1].fts!read"},
},
// case with invalid index name
{
method: http.MethodGet,
uri: "/api/index/x1",
path: "/api/index/{indexName}",
err: errIndexNotFound,
},
// case with invalid index name (actuall pindex name)
{
method: http.MethodGet,
uri: "/api/index/p1",
path: "/api/index/{indexName}",
err: errIndexNotFound,
},
// case with valid pindex name
{
method: http.MethodGet,
uri: "/api/pindex/p1",
path: "/api/pindex/{pindexName}",
perms: []string{"cluster.bucket[s3].fts!read"},
},
// case with invalid pindex name
{
method: http.MethodGet,
uri: "/api/pindex/y1",
path: "/api/pindex/{pindexName}",
err: errPIndexNotFound,
},
// case with invalid pindex name (actually index name)
{
method: http.MethodGet,
uri: "/api/pindex/i1",
path: "/api/pindex/{pindexName}",
err: errPIndexNotFound,
},
// case with valid alias, but this operation does NOT expand alias
{
method: http.MethodGet,
uri: "/api/index/a1",
path: "/api/index/{indexName}",
perms: []string{"cluster.bucket.fts!read"},
},
// case with valid alias, and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a1/count",
path: "/api/index/{indexName}/count",
perms: []string{"cluster.bucket[s1].fts!read"},
},
// case with valid alias (multi), and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a2/count",
path: "/api/index/{indexName}/count",
perms: []string{"cluster.bucket[s1].fts!read", "cluster.bucket[s2].fts!read"},
},
// case with valid alias (containing another alias),
// and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a3/count",
path: "/api/index/{indexName}/count",
perms: []string{"cluster.bucket[s1].fts!read", "cluster.bucket[s2].fts!read"},
},
// test special case for creating new index
{
method: http.MethodPut,
uri: "/api/index/anewone",
body: []byte(`{"type":"fulltext-index","sourceType":"couchbase","sourceName":"abucket"}`),
path: "/api/index/{indexName}",
perms: []string{"cluster.bucket[abucket].fts!write"},
},
}
for i, test := range tests {
var r io.Reader
if test.body != nil {
r = bytes.NewBuffer(test.body)
}
req, err := http.NewRequest(test.method, test.uri, r)
if err != nil {
t.Fatal(err)
}
// this actually executes things, which will usually fail
// which is unrelated to what we're testing
// but its the best i could do
record := httptest.NewRecorder()
router.ServeHTTP(record, req)
// set the request body again, as its been consumed :(
if test.body != nil {
req.Body = ioutil.NopCloser(bytes.NewBuffer(test.body))
}
actualPerms, err := preparePerms(s, req, test.method, test.path)
if err != test.err {
t.Errorf("test %d, expected err %v, got %v", i, test.err, err)
}
sort.Strings(actualPerms)
if !reflect.DeepEqual(actualPerms, test.perms) {
t.Errorf("test %d, expected %v, got %v", i, test.perms, actualPerms)
}
}
}
func TestHandlersForRuntimeOps(t *testing.T) {
emptyDir, err := ioutil.TempDir("./tmp", "test")
if err != nil {
t.Errorf("tempdir err: %v", err)
}
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
err = mgr.Start("wanted")
if err != nil {
t.Errorf("expected no start err, got: %v", err)
}
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
mr.Write([]byte("hello"))
mr.Write([]byte("world"))
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr,
AssetDir, Asset)
if err != nil || router == nil {
t.Errorf("no mux router")
}
tests := []*RESTHandlerTest{
{
Path: "/api/runtime",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
"arch": true,
"go": true,
"GOMAXPROCS": true,
"GOROOT": true,
},
},
{
Path: "/api/runtime/args",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
// Actual production args are different from "go test" context.
},
},
{
Path: "/api/runtime/gc",
Method: "POST",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseBody: []byte(nil),
},
{
Path: "/api/runtime/statsMem",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
"Alloc": true,
"TotalAlloc": true,
},
},
{
Path: "/api/runtime/profile/cpu",
Method: "POST",
Params: nil,
Body: nil,
Status: 400,
ResponseMatch: map[string]bool{
"incorrect or missing secs parameter": true,
},
},
}
testRESTHandlers(t, tests, router)
}
func TestSourceNamesFromReq(t *testing.T) {
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
mgr.Start("wanted")
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
if err != nil || router == nil {
t.Errorf("no mux router")
}
router.KeepContext = true // so we can see the mux vars
s := &stubDefinitionLookuper{
pindexes: testPIndexesByName,
defs: &cbgt.IndexDefs{
IndexDefs: testIndexDefsByName,
},
}
tests := []struct {
method string
uri string
path string
sources []string
err error
}{
// case with valid index name
{
method: http.MethodGet,
uri: "/api/index/i1",
path: "/api/index/{indexName}",
sources: []string{"s1"},
},
// case with invalid index name
{
method: http.MethodGet,
uri: "/api/index/x1",
path: "/api/index/{indexName}",
err: errIndexNotFound,
},
// case with invalid index name (actuall pindex name)
{
method: http.MethodGet,
uri: "/api/index/p1",
path: "/api/index/{indexName}",
err: errIndexNotFound,
},
// case with valid pindex name
{
method: http.MethodGet,
uri: "/api/pindex/p1",
path: "/api/pindex/{pindexName}",
sources: []string{"s3"},
},
// case with invalid pindex name
{
method: http.MethodGet,
uri: "/api/pindex/y1",
path: "/api/pindex/{pindexName}",
err: errPIndexNotFound,
},
// case with invalid pindex name (actually index name)
{
method: http.MethodGet,
uri: "/api/pindex/i1",
path: "/api/pindex/{pindexName}",
err: errPIndexNotFound,
},
// case with valid alias, but this operation does NOT expand alias
{
method: http.MethodGet,
uri: "/api/index/a1",
path: "/api/index/{indexName}",
sources: []string{},
},
// case with valid alias, and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a1/count",
path: "/api/index/{indexName}/count",
sources: []string{"s1"},
},
// case with valid alias (multi), and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a2/count",
path: "/api/index/{indexName}/count",
sources: []string{"s1", "s2"},
},
// case with valid alias (containing another alias),
// and this operation DOES expand alias
{
method: http.MethodGet,
uri: "/api/index/a3/count",
path: "/api/index/{indexName}/count",
sources: []string{"s1", "s2"},
},
}
for i, test := range tests {
req, err := http.NewRequest(test.method, test.uri, nil)
if err != nil {
t.Fatal(err)
}
// this actually executes things, which will usually fail
// which is unrelated to what we're testing
// but its the best i could do
record := httptest.NewRecorder()
router.ServeHTTP(record, req)
actualNames, err := sourceNamesFromReq(s, req, test.method, test.path)
if err != test.err {
t.Errorf("test %d, expected err %v, got %v", i, test.err, err)
}
sort.Strings(actualNames)
if !reflect.DeepEqual(actualNames, test.sources) {
t.Errorf("test %d, expected %v, got %v", i, test.sources, actualNames)
}
}
}
func main() {
flag.Parse()
if flags.Help {
flag.Usage()
os.Exit(2)
}
if flags.Version {
fmt.Printf("%s main: %s, data: %s\n", path.Base(os.Args[0]),
VERSION, cbgt.VERSION)
os.Exit(0)
}
if os.Getenv("GOMAXPROCS") == "" {
runtime.GOMAXPROCS(runtime.NumCPU())
}
mr, err := cbgt.NewMsgRing(os.Stderr, 1000)
if err != nil {
log.Fatalf("main: could not create MsgRing, err: %v", err)
}
log.SetOutput(mr)
log.SetLoggerCallback(LoggerFunc)
log.Printf("main: %s started (%s/%s)",
os.Args[0], VERSION, cbgt.VERSION)
rand.Seed(time.Now().UTC().UnixNano())
go cmd.DumpOnSignalForPlatform()
bleve.StoreDynamic = false
bleve.MappingJSONStrict = true
MainWelcome(flagAliases)
s, err := os.Stat(flags.DataDir)
if err != nil {
if os.IsNotExist(err) {
if flags.DataDir == DEFAULT_DATA_DIR {
log.Printf("main: creating data directory, dataDir: %s",
flags.DataDir)
err = os.Mkdir(flags.DataDir, 0700)
if err != nil {
log.Fatalf("main: could not create data directory,"+
" dataDir: %s, err: %v", flags.DataDir, err)
}
} else {
log.Fatalf("main: data directory does not exist,"+
" dataDir: %s", flags.DataDir)
return
}
} else {
log.Fatalf("main: could not access data directory,"+
" dataDir: %s, err: %v", flags.DataDir, err)
return
}
} else {
if !s.IsDir() {
log.Fatalf("main: not a directory, dataDir: %s", flags.DataDir)
return
}
}
cbft.SetAuthType(flags.AuthType)
wd, err := os.Getwd()
if err != nil {
log.Fatalf("main: os.Getwd, err: %#v", err)
return
}
log.Printf("main: curr dir: %q", wd)
dataDirAbs, err := filepath.Abs(flags.DataDir)
if err != nil {
log.Fatalf("main: filepath.Abs, err: %#v", err)
return
}
log.Printf("main: data dir: %q", dataDirAbs)
// User may supply a comma-separated list of HOST:PORT values for
// http addresss/port listening, but only the first entry is used
// for cbgt node and Cfg registration.
bindHttps := strings.Split(flags.BindHttp, ",")
// If cfg is down, we error, leaving it to some user-supplied
// outside watchdog to backoff and restart/retry.
cfg, err := cmd.MainCfg(cmdName, flags.CfgConnect,
bindHttps[0], flags.Register, flags.DataDir)
if err != nil {
if err == cmd.ErrorBindHttp {
log.Fatalf("%v", err)
return
}
log.Fatalf("main: could not start cfg, cfgConnect: %s, err: %v\n"+
" Please check that your -cfg/-cfgConnect parameter (%q)\n"+
" is correct and/or that your configuration provider\n"+
" is available.",
flags.CfgConnect, err, flags.CfgConnect)
return
}
uuid := flags.UUID
if uuid != "" {
uuidPath :=
flags.DataDir + string(os.PathSeparator) + cmdName + ".uuid"
err = ioutil.WriteFile(uuidPath, []byte(uuid), 0600)
if err != nil {
log.Fatalf("main: could not write uuidPath: %s\n"+
" Please check that your -data/-dataDir parameter (%q)\n"+
" is to a writable directory where %s can persist data.",
uuidPath, flags.DataDir, cmdName)
return
}
}
if uuid == "" {
uuid, err = cmd.MainUUID(cmdName, flags.DataDir)
if err != nil {
log.Fatalf(fmt.Sprintf("%v", err))
return
}
}
var tagsArr []string
if flags.Tags != "" {
tagsArr = strings.Split(flags.Tags, ",")
}
router, err := MainStart(cfg, uuid, tagsArr,
flags.Container, flags.Weight, flags.Extras,
bindHttps[0], flags.DataDir,
flags.StaticDir, flags.StaticETag,
flags.Server, flags.Register, mr, flags.Options)
if err != nil {
log.Fatal(err)
}
if flags.Register == "unknown" {
log.Printf("main: unregistered node; now exiting")
os.Exit(0)
}
http.Handle("/", router)
anyHostPorts := map[string]bool{}
// Bind to 0.0.0.0's first.
for _, bindHttp := range bindHttps {
if strings.HasPrefix(bindHttp, "0.0.0.0:") {
go MainServeHttp(bindHttp, nil)
anyHostPorts[bindHttp] = true
}
}
for i := len(bindHttps) - 1; i >= 1; i-- {
go MainServeHttp(bindHttps[i], anyHostPorts)
}
MainServeHttp(bindHttps[0], anyHostPorts)
<-(make(chan struct{})) // Block forever.
}
func testHandlersWithOnePartitionPrimaryFeedPartialRollbackMoss(t *testing.T,
rollbackToSeq uint64,
rollbackDesc string,
afterRollbackCountResponseMatch map[string]bool) {
BlevePIndexAllowMossPrev := BlevePIndexAllowMoss
BlevePIndexAllowMoss = true
bleveConfigDefaultKVStorePrev := bleve.Config.DefaultKVStore
bleve.Config.DefaultKVStore = "mossStore"
bleveMossRegistryCollectionOptionsFTSPrev := bleveMoss.RegistryCollectionOptions["fts"]
bleveMoss.RegistryCollectionOptions["fts"] = moss.CollectionOptions{}
defer func() {
BlevePIndexAllowMoss = BlevePIndexAllowMossPrev
bleve.Config.DefaultKVStore = bleveConfigDefaultKVStorePrev
bleveMoss.RegistryCollectionOptions["fts"] = bleveMossRegistryCollectionOptionsFTSPrev
}()
emptyDir, _ := ioutil.TempDir("./tmp", "test")
defer os.RemoveAll(emptyDir)
cfg := cbgt.NewCfgMem()
meh := &TestMEH{}
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
mgr.Start("wanted")
mgr.Kick("test-start-kick")
mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
router, _, err := NewRESTRouter("v0", mgr, "static", "", mr, nil)
if err != nil || router == nil {
t.Errorf("no mux router")
}
var pindex *cbgt.PIndex
var feed *cbgt.PrimaryFeed
waitForPersistence := func() {
for i := 0; i < 100; i++ {
stats := map[string]interface{}{
"num_recs_to_persist": float64(0),
}
err := addPIndexStats(pindex, stats)
if err != nil {
t.Errorf("expected nil addPIndexStats err, got: %v", err)
}
v, ok := stats["num_recs_to_persist"]
if ok {
nrtp, ok := v.(float64)
if ok && nrtp <= 0.0 {
return
}
}
time.Sleep(50 * time.Millisecond)
}
t.Errorf("persistence took too long!")
}
tests := []*RESTHandlerTest{
{
Desc: "create dest feed with 1 partition for rollback",
Path: "/api/index/idx0",
Method: "PUT",
Params: url.Values{
"indexType": []string{"fulltext-index"},
"indexParams": []string{
`{"store":{"mossStoreOptions":{"CompactionPercentage":100000.0}}}`,
}, // Never compact during this test.
"sourceType": []string{"primary"},
"sourceParams": []string{`{"numPartitions":1}`},
},
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok"}`: true,
},
After: func() {
time.Sleep(10 * time.Millisecond)
mgr.Kick("after-rollback")
feeds, pindexes := mgr.CurrentMaps()
if len(feeds) != 1 {
t.Errorf("expected to be 1 feed, got feeds: %+v", feeds)
}
if len(pindexes) != 1 {
t.Errorf("expected to be 1 pindex,"+
" got pindexes: %+v", pindexes)
}
for _, f := range feeds {
var ok bool
feed, ok = f.(*cbgt.PrimaryFeed)
if !ok {
t.Errorf("expected the 1 feed to be a PrimaryFeed")
}
}
for _, p := range pindexes {
pindex = p
}
},
},
{
Before: func() {
partition := "0"
snapStart := uint64(1)
snapEnd := uint64(10)
err = feed.SnapshotStart(partition, snapStart, snapEnd)
if err != nil {
t.Errorf("expected no err on snapshot-start")
}
},
Desc: "count idx0 0 when snapshot just started, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":0}`: true,
},
},
{
Before: func() {
partition := "0"
key := []byte("ss1-k1")
seq := uint64(1)
val := []byte(`{"foo":"bar","yow":"wow"}`)
err = feed.DataUpdate(partition, key, seq, val,
0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
if err != nil {
t.Errorf("expected no err on data-update")
}
},
Desc: "count idx0 should be 0 when got an update (doc created)" +
" but snapshot not ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":0}`: true,
},
},
{
Before: func() {
partition := "0"
key := []byte("ss1-k2")
seq := uint64(2)
val := []byte(`{"foo":"bing","yow":"wow"}`)
err = feed.DataUpdate(partition, key, seq, val,
0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
if err != nil {
t.Errorf("expected no err on data-update")
}
},
Desc: "count idx0 0 when got 2nd update (another doc created)" +
" but snapshot not ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":0}`: true,
},
},
{
Before: func() {
partition := "0"
snapStart := uint64(11)
snapEnd := uint64(20)
err = feed.SnapshotStart(partition, snapStart, snapEnd)
if err != nil {
t.Errorf("expected no err on snapshot-start")
}
waitForPersistence()
},
Desc: "count idx0 2 when 1st snapshot ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":2}`: true,
},
},
{
Before: func() {
partition := "0"
key := []byte("ss11-k11")
seq := uint64(11)
val := []byte(`{"foo":"baz","yow":"NOPE"}`)
err = feed.DataUpdate(partition, key, seq, val,
0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
if err != nil {
t.Errorf("expected no err on data-update")
}
},
Desc: "count idx0 should be 2 when got an update (doc created)" +
" in 2nd snapshot, 2nd snapshot not ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":2}`: true,
},
},
{
Before: func() {
partition := "0"
snapStart := uint64(21)
snapEnd := uint64(30)
err = feed.SnapshotStart(partition, snapStart, snapEnd)
if err != nil {
t.Errorf("expected no err on snapshot-start")
}
waitForPersistence()
},
Desc: "count idx0 3 when 2nd snapshot ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":3}`: true,
},
},
{
Before: func() {
partition := "0"
key := []byte("ss21-k21")
seq := uint64(21)
val := []byte(`{"foo":"baz","yow":"whoa"}`)
err = feed.DataUpdate(partition, key, seq, val,
0, cbgt.DEST_EXTRAS_TYPE_NIL, nil)
if err != nil {
t.Errorf("expected no err on data-update")
}
},
Desc: "count idx0 should be 3 when got an update (doc created)" +
" in 3rd snapshot, 3rd snapshot not ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":3}`: true,
},
},
{
Before: func() {
partition := "0"
snapStart := uint64(31)
snapEnd := uint64(40)
err = feed.SnapshotStart(partition, snapStart, snapEnd)
if err != nil {
t.Errorf("expected no err on snapshot-start")
}
waitForPersistence()
},
Desc: "count idx0 4 when 3rd snapshot ended, pre-rollback",
Path: "/api/index/idx0/count",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: map[string]bool{
`{"status":"ok","count":4}`: true,
},
},
{
// Rollback to a high seq number, so it's a no-op
// w.r.t. doing an actual rollback.
Before: func() {
partition := "0"
err = feed.Rollback(partition, rollbackToSeq)
if err != nil {
t.Errorf("expected no err on rollback, got: %v", err)
}
// NOTE: We might check right after rollback but before
// we get a kick, but unfortunately results will be race-y.
time.Sleep(100 * time.Millisecond)
mgr.Kick("after-rollback")
},
Desc: rollbackDesc,
Path: "/api/index/idx0/count",
Method: "GET",
Params: nil,
Body: nil,
Status: http.StatusOK,
ResponseMatch: afterRollbackCountResponseMatch,
},
}
testRESTHandlers(t, tests, router)
}
// Emits markdown docs of cbft's REST API.
func main() {
rand.Seed(0)
dataDir, _ := ioutil.TempDir("./tmp", "data")
defer os.RemoveAll(dataDir)
cfg := cbgt.NewCfgMem()
tags := []string(nil)
container := ""
weight := 1
extras := ""
bindHttp := "0.0.0.0:8095"
mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
tags, container, weight, extras, bindHttp,
dataDir, "http://localhost:8091", nil)
sourceType := "nil"
sourceName := ""
sourceUUID := ""
sourceParams := ""
indexType := "blackhole"
indexName := "myFirstIndex"
indexParams := ""
prevIndexUUID := ""
mgr.Start("wanted")
err := mgr.CreateIndex(
sourceType, sourceName, sourceUUID, sourceParams,
indexType, indexName, indexParams,
cbgt.PlanParams{}, prevIndexUUID)
if err != nil {
log.Fatalf("could not create myFirstIndex, err: %v", err)
}
staticDir := ""
staticETag := ""
mr, _ := cbgt.NewMsgRing(ioutil.Discard, 1)
router, meta, err :=
cbft.NewRESTRouter(VERSION, mgr, staticDir, staticETag, mr)
if err != nil {
log.Panic(err)
}
mainCategoriesMap := map[string]bool{}
mainCategories := []string(nil)
subCategoriesMap := map[string]bool{}
subCategories := []string(nil)
paths := []string(nil)
for path := range meta {
paths = append(paths, path)
m := meta[path]
if m.Opts != nil {
category := m.Opts["_category"]
mainCategory, _, subCategory, _ := categoryParse(category)
if !mainCategoriesMap[mainCategory] {
mainCategoriesMap[mainCategory] = true
mainCategories = append(mainCategories, mainCategory)
}
if !subCategoriesMap[subCategory] {
subCategoriesMap[subCategory] = true
subCategories = append(subCategories, subCategory)
}
}
}
sort.Strings(mainCategories)
sort.Strings(subCategories)
sort.Strings(paths)
fmt.Printf("# API Reference\n\n")
for _, mainCategory := range mainCategories {
mainCategoryFirst := true
for _, subCategory := range subCategories {
subCategoryFirst := true
for _, path := range paths {
m := meta[path]
category := ""
if m.Opts != nil {
category = m.Opts["_category"]
}
mc, mcVis, sc, scVis := categoryParse(category)
if mc != mainCategory ||
sc != subCategory {
continue
}
if m.Opts != nil &&
m.Opts["_status"] == "private" {
continue
}
if mainCategoryFirst {
fmt.Printf("---\n\n# %s\n\n", mcVis)
}
mainCategoryFirst = false
if subCategoryFirst {
fmt.Printf("---\n\n## %s\n\n", scVis)
}
subCategoryFirst = false
pathParts := strings.Split(path, " ")
fmt.Printf("---\n\n")
fmt.Printf("%s `%s`\n\n", m.Method, pathParts[0])
if m.Opts != nil && m.Opts["_about"] != "" {
fmt.Printf("%s\n\n", m.Opts["_about"])
}
optNames := []string(nil)
for optName := range m.Opts {
if optName != "" && !strings.HasPrefix(optName, "_") {
optNames = append(optNames, optName)
}
}
sort.Strings(optNames)
for _, optName := range optNames {
fmt.Printf("**%s**: %s\n\n", optName, m.Opts[optName])
}
if m.Opts != nil && m.Opts[""] != "" {
fmt.Printf("%s\n\n", m.Opts[""])
}
if m.Method == "GET" && !skipSampleResponses[pathParts[0]] {
url := strings.Replace(pathParts[0],
"{indexName}", "myFirstIndex", 1)
res := sampleRequest(router, m.Method, url, nil)
if res != nil {
var j map[string]interface{}
err = json.Unmarshal(res, &j)
if err == nil {
s, err := json.MarshalIndent(j, " ", " ")
if err == nil {
fmt.Printf("Sample response:\n\n %s\n\n", s)
}
}
}
}
}
}
}
fmt.Printf("---\n\nCopyright (c) 2015 Couchbase, Inc.\n")
}
func main() {
platform.HideConsole(true)
defer platform.HideConsole(false)
flag.Parse()
if flags.Help {
flag.Usage()
os.Exit(2)
}
if flags.Version {
fmt.Printf("%s main: %s, data: %s\n", path.Base(os.Args[0]),
VERSION, cbgt.VERSION)
os.Exit(0)
}
if os.Getenv("GOMAXPROCS") == "" {
runtime.GOMAXPROCS(runtime.NumCPU())
}
mr, err := cbgt.NewMsgRing(os.Stderr, 1000)
if err != nil {
log.Fatalf("main: could not create MsgRing, err: %v", err)
}
log.SetOutput(mr)
log.SetLoggerCallback(LoggerFunc)
log.Printf("main: %s started (%s/%s)",
os.Args[0], VERSION, cbgt.VERSION)
rand.Seed(time.Now().UTC().UnixNano())
// disabling as speculative change for
// https://issues.couchbase.com/browse/MB-20002
// theory being that invoking signal.Notify()
// interferes with forestdb signal handler
// go cmd.DumpOnSignalForPlatform()
MainWelcome(flagAliases)
s, err := os.Stat(flags.DataDir)
if err != nil {
if os.IsNotExist(err) {
if flags.DataDir == DEFAULT_DATA_DIR {
log.Printf("main: creating data directory, dataDir: %s",
flags.DataDir)
err = os.Mkdir(flags.DataDir, 0700)
if err != nil {
log.Fatalf("main: could not create data directory,"+
" dataDir: %s, err: %v", flags.DataDir, err)
}
} else {
log.Fatalf("main: data directory does not exist,"+
" dataDir: %s", flags.DataDir)
return
}
} else {
log.Fatalf("main: could not access data directory,"+
" dataDir: %s, err: %v", flags.DataDir, err)
return
}
} else {
if !s.IsDir() {
log.Fatalf("main: not a directory, dataDir: %s", flags.DataDir)
return
}
}
wd, err := os.Getwd()
if err != nil {
log.Fatalf("main: os.Getwd, err: %#v", err)
return
}
log.Printf("main: curr dir: %q", wd)
dataDirAbs, err := filepath.Abs(flags.DataDir)
if err != nil {
log.Fatalf("main: filepath.Abs, err: %#v", err)
return
}
log.Printf("main: data dir: %q", dataDirAbs)
uuid := flags.UUID
if uuid != "" {
uuidPath :=
flags.DataDir + string(os.PathSeparator) + cmdName + ".uuid"
err = ioutil.WriteFile(uuidPath, []byte(uuid), 0600)
if err != nil {
log.Fatalf("main: could not write uuidPath: %s\n"+
" Please check that your -data/-dataDir parameter (%q)\n"+
" is to a writable directory where %s can persist data.",
uuidPath, flags.DataDir, cmdName)
return
}
}
if uuid == "" {
uuid, err = cmd.MainUUID(cmdName, flags.DataDir)
if err != nil {
log.Fatalf("%v", err)
return
}
}
options := cmd.ParseOptions(flags.Options, "CBFT_ENV_OPTIONS",
map[string]string{
"managerLoadDataDir": "async",
"authType": flags.AuthType,
})
err = InitHttpOptions(options)
if err != nil {
log.Fatalf("main: InitHttpOptions, err: %v", err)
return
}
// User may supply a comma-separated list of HOST:PORT values for
// http addresss/port listening, but only the first http entry
// is used for cbgt node and Cfg registration.
bindHttpList := strings.Split(flags.BindHttp, ",")
// If cfg is down, we error, leaving it to some user-supplied
// outside watchdog to backoff and restart/retry.
cfg, err := cmd.MainCfgEx(cmdName, flags.CfgConnect,
bindHttpList[0], flags.Register, flags.DataDir, uuid, options)
if err != nil {
if err == cmd.ErrorBindHttp {
log.Fatalf("%v", err)
return
}
log.Fatalf("main: could not start cfg, cfgConnect: %s, err: %v\n"+
" Please check that your -cfg/-cfgConnect parameter (%q)\n"+
" is correct and/or that your configuration provider\n"+
" is available.",
flags.CfgConnect, err, flags.CfgConnect)
return
}
var tagsArr []string
if flags.Tags != "" {
tagsArr = strings.Split(flags.Tags, ",")
}
router, err := MainStart(cfg, uuid, tagsArr,
flags.Container, flags.Weight, flags.Extras,
bindHttpList[0], flags.DataDir,
flags.StaticDir, flags.StaticETag,
flags.Server, flags.Register, mr, options)
if err != nil {
log.Fatal(err)
}
if flags.Register == "unknown" {
log.Printf("main: unregistered node; now exiting")
os.Exit(0)
}
http.Handle("/", router)
anyHostPorts := map[string]bool{}
// Bind to 0.0.0.0's first for http listening.
for _, bindHttp := range bindHttpList {
if strings.HasPrefix(bindHttp, "0.0.0.0:") {
go MainServeHttp("http", bindHttp, nil, "", "")
anyHostPorts[bindHttp] = true
}
}
for i := len(bindHttpList) - 1; i >= 1; i-- {
go MainServeHttp("http", bindHttpList[i], anyHostPorts, "", "")
}
if flags.BindHttps != "" {
bindHttpsList := strings.Split(flags.BindHttps, ",")
// Bind to 0.0.0.0's first for https listening.
for _, bindHttps := range bindHttpsList {
if strings.HasPrefix(bindHttps, "0.0.0.0:") {
go MainServeHttp("https", bindHttps, nil,
flags.TlsCertFile, flags.TlsKeyFile)
anyHostPorts[bindHttps] = true
}
}
for _, bindHttps := range bindHttpsList {
go MainServeHttp("https", bindHttps, anyHostPorts,
flags.TlsCertFile, flags.TlsKeyFile)
}
}
MainServeHttp("http", bindHttpList[0], anyHostPorts, "", "")
<-(make(chan struct{})) // Block forever.
}