func InitStaticRouter(staticDir, staticETag string,
mgr *cbgt.Manager) *mux.Router {
prefix := ""
if mgr != nil {
prefix = mgr.Options()["urlPrefix"]
}
hfsStaticX := http.FileServer(assetFS())
router := mux.NewRouter()
router.StrictSlash(true)
router.Handle(prefix+"/",
http.RedirectHandler(prefix+"/index.html", 302))
router.Handle(prefix+"/index.html",
http.RedirectHandler(prefix+"/staticx/index.html", 302))
router.Handle(prefix+"/static/partials/index/start.html",
http.RedirectHandler(prefix+"/staticx/partials/index/start.html", 302))
router = rest.InitStaticRouterEx(router,
staticDir, staticETag, []string{
prefix + "/indexes",
prefix + "/nodes",
prefix + "/monitor",
prefix + "/manage",
prefix + "/logs",
prefix + "/debug",
}, http.RedirectHandler(prefix+"/staticx/index.html", 302), mgr)
router.PathPrefix(prefix + "/staticx/").Handler(
http.StripPrefix(prefix+"/staticx/", hfsStaticX))
return router
}
func BlevePIndexImplInitRouter(r *mux.Router, phase string,
mgr *cbgt.Manager) {
prefix := ""
if mgr != nil {
prefix = mgr.Options()["urlPrefix"]
}
if phase == "static.before" {
staticBleveMapping := http.FileServer(bleveMappingUI.AssetFS())
r.PathPrefix(prefix + "/static-bleve-mapping/").Handler(
http.StripPrefix(prefix+"/static-bleve-mapping/",
staticBleveMapping))
bleveMappingUI.RegisterHandlers(r, prefix+"/api")
}
if phase == "manager.after" {
// Using standard bleveHttp handlers for /api/pindex-bleve endpoints.
//
listIndexesHandler := bleveHttp.NewListIndexesHandler()
r.Handle(prefix+"/api/pindex-bleve",
listIndexesHandler).Methods("GET")
getIndexHandler := bleveHttp.NewGetIndexHandler()
getIndexHandler.IndexNameLookup = rest.PIndexNameLookup
r.Handle(prefix+"/api/pindex-bleve/{pindexName}",
getIndexHandler).Methods("GET")
docCountHandler := bleveHttp.NewDocCountHandler("")
docCountHandler.IndexNameLookup = rest.PIndexNameLookup
r.Handle(prefix+"/api/pindex-bleve/{pindexName}/count",
docCountHandler).Methods("GET")
searchHandler := bleveHttp.NewSearchHandler("")
searchHandler.IndexNameLookup = rest.PIndexNameLookup
r.Handle(prefix+"/api/pindex-bleve/{pindexName}/query",
searchHandler).Methods("POST")
docGetHandler := bleveHttp.NewDocGetHandler("")
docGetHandler.IndexNameLookup = rest.PIndexNameLookup
docGetHandler.DocIDLookup = rest.DocIDLookup
r.Handle(prefix+"/api/pindex-bleve/{pindexName}/doc/{docID}",
docGetHandler).Methods("GET")
debugDocHandler := bleveHttp.NewDebugDocumentHandler("")
debugDocHandler.IndexNameLookup = rest.PIndexNameLookup
debugDocHandler.DocIDLookup = rest.DocIDLookup
r.Handle(prefix+"/api/pindex-bleve/{pindexName}/docDebug/{docID}",
debugDocHandler).Methods("GET")
listFieldsHandler := bleveHttp.NewListFieldsHandler("")
listFieldsHandler.IndexNameLookup = rest.PIndexNameLookup
r.Handle(prefix+"/api/pindex-bleve/{pindexName}/fields",
listFieldsHandler).Methods("GET")
}
}
// InitStaticRouterEx is like InitStaticRouter, but with optional
// manager parameter for more options.
func InitStaticRouterEx(r *mux.Router, staticDir, staticETag string,
pages []string, pagesHandler http.Handler,
mgr *cbgt.Manager) *mux.Router {
prefix := ""
if mgr != nil {
prefix = mgr.Options()["urlPrefix"]
}
PIndexTypesInitRouter(r, "static.before", mgr)
var s http.FileSystem
if staticDir != "" {
if _, err := os.Stat(staticDir); err == nil {
log.Printf("http: serving assets from staticDir: %s", staticDir)
s = http.Dir(staticDir)
}
}
if s == nil {
log.Printf("http: serving assets from embedded data")
s = AssetFS()
}
r.PathPrefix(prefix + "/static/").Handler(
http.StripPrefix(prefix+"/static/",
ETagFileHandler{http.FileServer(s), staticETag}))
// Bootstrap UI insists on loading templates from this path.
r.PathPrefix(prefix + "/template/").Handler(
http.StripPrefix(prefix+"/template/",
ETagFileHandler{http.FileServer(s), staticETag}))
// If client ask for any of the pages, redirect.
for _, p := range pages {
if pagesHandler != nil {
r.PathPrefix(p).Handler(pagesHandler)
} else {
r.PathPrefix(p).Handler(RewriteURL("/", http.FileServer(s)))
}
}
r.Handle(prefix+"/index.html",
http.RedirectHandler(prefix+"/static/index.html", 302))
r.Handle(prefix+"/",
http.RedirectHandler(prefix+"/static/index.html", 302))
PIndexTypesInitRouter(r, "static.after", mgr)
return r
}
func NewQueryHandler(mgr *cbgt.Manager) *QueryHandler {
slowQueryLogTimeout := time.Duration(0)
slowQueryLogTimeoutV := mgr.Options()["slowQueryLogTimeout"]
if slowQueryLogTimeoutV != "" {
d, err := time.ParseDuration(slowQueryLogTimeoutV)
if err == nil {
slowQueryLogTimeout = d
}
}
return &QueryHandler{
mgr: mgr,
slowQueryLogTimeout: slowQueryLogTimeout,
}
}
// Returns a bleve.IndexAlias that represents all the PIndexes for the
// index, including perhaps bleve remote client PIndexes.
//
// TODO: Perhaps need a tighter check around indexUUID, as the current
// implementation might have a race where old pindexes with a matching
// (but invalid) indexUUID might be hit.
//
// TODO: If this returns an error, perhaps the caller somewhere up the
// chain should close the cancelCh to help stop any other inflight
// activities.
func bleveIndexAlias(mgr *cbgt.Manager, indexName, indexUUID string,
ensureCanRead bool, consistencyParams *cbgt.ConsistencyParams,
cancelCh <-chan bool) (bleve.IndexAlias, error) {
planPIndexNodeFilter := cbgt.PlanPIndexNodeOk
if ensureCanRead {
planPIndexNodeFilter = cbgt.PlanPIndexNodeCanRead
}
localPIndexes, remotePlanPIndexes, err :=
mgr.CoveringPIndexes(indexName, indexUUID, planPIndexNodeFilter,
"queries")
if err != nil {
return nil, fmt.Errorf("bleve: bleveIndexAlias, err: %v", err)
}
alias := bleve.NewIndexAlias()
for _, remotePlanPIndex := range remotePlanPIndexes {
baseURL := "http://" + remotePlanPIndex.NodeDef.HostPort +
"/api/pindex/" + remotePlanPIndex.PlanPIndex.Name
alias.Add(&IndexClient{
QueryURL: baseURL + "/query",
CountURL: baseURL + "/count",
Consistency: consistencyParams,
// TODO: Propagate auth to remote client.
})
}
// TODO: Should kickoff remote queries concurrently before we wait.
err = cbgt.ConsistencyWaitGroup(indexName, consistencyParams,
cancelCh, localPIndexes,
func(localPIndex *cbgt.PIndex) error {
bindex, ok := localPIndex.Impl.(bleve.Index)
if !ok || bindex == nil ||
!strings.HasPrefix(localPIndex.IndexType, "bleve") {
return fmt.Errorf("bleve: wrong type, localPIndex: %#v",
localPIndex)
}
alias.Add(bindex)
return nil
})
if err != nil {
return nil, err
}
return alias, nil
}
func CheckAPIAuth(mgr *cbgt.Manager,
w http.ResponseWriter, req *http.Request, path string) (allowed bool) {
authType := ""
if mgr != nil && mgr.Options() != nil {
authType = mgr.Options()["authType"]
}
if authType == "" {
return true
}
if authType != "cbauth" {
return false
}
creds, err := cbauth.AuthWebCreds(req)
if err != nil {
http.Error(w, fmt.Sprintf("rest_auth: cbauth.AuthWebCreds,"+
" err: %v ", err), 403)
return false
}
perms, err := preparePerms(mgr, req, req.Method, path)
if err != nil {
http.Error(w, fmt.Sprintf("rest_auth: preparePerm,"+
" err: %v ", err), 403)
return false
}
for _, perm := range perms {
allowed, err = creds.IsAllowed(perm)
if err != nil {
http.Error(w, fmt.Sprintf("rest_auth: cbauth.IsAllowed,"+
" err: %v ", err), 403)
return false
}
if !allowed {
cbauth.SendForbidden(w, perm)
return false
}
}
return true
}
func (k *kvChangeIndex) retrieveCBGTPartitions() (partitionDef base.PartitionStorageSet, err error) {
var manager *cbgt.Manager
if k.context != nil {
manager = k.context.BucketSpec.CbgtContext.Manager
} else {
return nil, errors.New("Unable to retrieve CBGT partitions - no database context")
}
if manager == nil {
return nil, errors.New("Unable to retrieve CBGT partitions - no CBGT manager")
}
_, planPIndexesByName, _ := manager.GetPlanPIndexes(true)
indexName := k.context.GetCBGTIndexNameForBucket(k.context.Bucket)
pindexes := planPIndexesByName[indexName]
for _, pIndex := range pindexes {
vbStrings := strings.Split(pIndex.SourcePartitions, ",")
// convert string vbNos to uint16
vbNos := make([]uint16, len(vbStrings))
for i := 0; i < len(vbStrings); i++ {
vbNumber, err := strconv.ParseUint(vbStrings[i], 10, 16)
if err != nil {
base.LogFatal("Error creating index partition definition - unable to parse vbucket number %s as integer:%v", vbStrings[i], err)
}
vbNos[i] = uint16(vbNumber)
}
entry := base.PartitionStorage{
Index: uint16(0), // see below for index assignment
Uuid: pIndex.UUID,
VbNos: vbNos,
}
partitionDef = append(partitionDef, entry)
}
// NOTE: the ordering of pindexes returned by manager.GetPlanPIndexes isn't fixed (it's doing a map iteration somewhere).
// The mapping from UUID to VbNos will always be consistent. Sorting by UUID to maintain a consistent index ordering,
// then assigning index values.
partitionDef.Sort()
for i := 0; i < len(partitionDef); i++ {
partitionDef[i].Index = uint16(i)
}
return partitionDef, nil
}
func bleveIndexTargets(mgr *cbgt.Manager, indexName, indexUUID string,
ensureCanRead bool, consistencyParams *cbgt.ConsistencyParams,
cancelCh <-chan bool, collector BleveIndexCollector) error {
planPIndexNodeFilter := cbgt.PlanPIndexNodeOk
if ensureCanRead {
planPIndexNodeFilter = cbgt.PlanPIndexNodeCanRead
}
localPIndexes, remotePlanPIndexes, err :=
mgr.CoveringPIndexes(indexName, indexUUID, planPIndexNodeFilter,
"queries")
if err != nil {
return fmt.Errorf("bleve: bleveIndexTargets, err: %v", err)
}
prefix := mgr.Options()["urlPrefix"]
for _, remotePlanPIndex := range remotePlanPIndexes {
baseURL := "http://" + remotePlanPIndex.NodeDef.HostPort +
prefix + "/api/pindex/" + remotePlanPIndex.PlanPIndex.Name
collector.Add(&IndexClient{
name: fmt.Sprintf("IndexClient - %s", baseURL),
QueryURL: baseURL + "/query",
CountURL: baseURL + "/count",
Consistency: consistencyParams,
// TODO: Propagate auth to remote client.
})
}
// TODO: Should kickoff remote queries concurrently before we wait.
return cbgt.ConsistencyWaitGroup(indexName, consistencyParams,
cancelCh, localPIndexes,
func(localPIndex *cbgt.PIndex) error {
bindex, ok := localPIndex.Impl.(bleve.Index)
if !ok || bindex == nil ||
!strings.HasPrefix(localPIndex.IndexType, "fulltext-index") {
return fmt.Errorf("bleve: wrong type, localPIndex: %#v",
localPIndex)
}
collector.Add(bindex)
return nil
})
}
// NewRESTRouter creates a mux.Router initialized with the REST API
// and web UI routes. See also InitStaticRouter and InitRESTRouter if
// you need finer control of the router initialization.
func NewRESTRouter(versionMain string, mgr *cbgt.Manager,
staticDir, staticETag string, mr *cbgt.MsgRing,
assetDir func(name string) ([]string, error),
asset func(name string) ([]byte, error)) (
*mux.Router, map[string]RESTMeta, error) {
prefix := mgr.Options()["urlPrefix"]
r := mux.NewRouter()
r.StrictSlash(true)
r = InitStaticRouterEx(r,
staticDir, staticETag, []string{
prefix + "/indexes",
prefix + "/nodes",
prefix + "/monitor",
prefix + "/manage",
prefix + "/logs",
prefix + "/debug",
}, nil, mgr)
return InitRESTRouter(r, versionMain, mgr,
staticDir, staticETag, mr, assetDir, asset)
}
func InitStaticRouter(staticDir, staticETag string,
mgr *cbgt.Manager) *mux.Router {
router := mux.NewRouter()
router.StrictSlash(true)
showUI := true
if mgr != nil && mgr.Options()["hideUI"] != "" {
hideUI, err := strconv.ParseBool(mgr.Options()["hideUI"])
if err == nil && hideUI {
showUI = false
}
}
if showUI {
prefix := ""
if mgr != nil {
prefix = mgr.Options()["urlPrefix"]
}
hfsStaticX := http.FileServer(assetFS())
router.Handle(prefix+"/",
http.RedirectHandler(prefix+"/index.html", 302))
router.Handle(prefix+"/index.html",
http.RedirectHandler(prefix+"/staticx/index.html", 302))
router.Handle(prefix+"/static/partials/index/start.html",
http.RedirectHandler(prefix+"/staticx/partials/index/start.html", 302))
router = rest.InitStaticRouterEx(router,
staticDir, staticETag, []string{
prefix + "/indexes",
prefix + "/nodes",
prefix + "/monitor",
prefix + "/manage",
prefix + "/logs",
prefix + "/debug",
}, http.RedirectHandler(prefix+"/staticx/index.html", 302), mgr)
router.PathPrefix(prefix + "/staticx/").Handler(
http.StripPrefix(prefix+"/staticx/", hfsStaticX))
}
return router
}
// WriteManagerStatsJSON writes JSON stats for a manager, and is
// optionally focus'able on a particular indexName.
func WriteManagerStatsJSON(mgr *cbgt.Manager, w io.Writer,
indexName string) error {
feeds, pindexes := mgr.CurrentMaps()
feedNames := make([]string, 0, len(feeds))
for feedName := range feeds {
feedNames = append(feedNames, feedName)
}
sort.Strings(feedNames)
pindexNames := make([]string, 0, len(pindexes))
for pindexName := range pindexes {
pindexNames = append(pindexNames, pindexName)
}
sort.Strings(pindexNames)
feedStats := make(map[string][]byte)
for _, feedName := range feedNames {
var buf bytes.Buffer
err := feeds[feedName].Stats(&buf)
if err != nil {
return fmt.Errorf("feed stats err: %v", err)
}
feedStats[feedName] = buf.Bytes()
}
pindexStats := make(map[string][]byte)
for _, pindexName := range pindexNames {
var buf bytes.Buffer
err := pindexes[pindexName].Dest.Stats(&buf)
if err != nil {
return fmt.Errorf("pindex stats err: %v", err)
}
pindexStats[pindexName] = buf.Bytes()
}
w.Write(cbgt.JsonOpenBrace)
first := true
w.Write(statsFeedsPrefix)
for _, feedName := range feedNames {
if indexName == "" || indexName == feeds[feedName].IndexName() {
if !first {
w.Write(cbgt.JsonComma)
}
first = false
w.Write(statsNamePrefix)
w.Write([]byte(feedName))
w.Write(statsNameSuffix)
w.Write(feedStats[feedName])
}
}
w.Write(cbgt.JsonCloseBraceComma)
first = true
w.Write(statsPIndexesPrefix)
for _, pindexName := range pindexNames {
if indexName == "" || indexName == pindexes[pindexName].IndexName {
if !first {
w.Write(cbgt.JsonComma)
}
first = false
w.Write(statsNamePrefix)
w.Write([]byte(pindexName))
w.Write(statsNameSuffix)
w.Write(pindexStats[pindexName])
}
}
w.Write(cbgt.JsonCloseBrace)
if indexName == "" {
w.Write(statsManagerPrefix)
var mgrStats cbgt.ManagerStats
mgr.StatsCopyTo(&mgrStats)
mgrStatsJSON, err := json.Marshal(&mgrStats)
if err == nil && len(mgrStatsJSON) > 0 {
w.Write(mgrStatsJSON)
} else {
w.Write(cbgt.JsonNULL)
}
}
w.Write(cbgt.JsonCloseBrace)
return nil
}
func (k *kvChangeIndex) initIndexPartitions() (*base.IndexPartitions, error) {
k.indexPartitionsLock.Lock()
defer k.indexPartitionsLock.Unlock()
// Check if it's been initialized while we waited for the lock
if k.indexPartitions != nil {
return k.indexPartitions, nil
}
var partitionDef []base.PartitionStorage
// First attempt to load from the bucket
value, _, err := k.reader.indexReadBucket.GetRaw(base.KIndexPartitionKey)
indexExpvars.Add("get_indexPartitionMap", 1)
if err == nil {
if err = json.Unmarshal(value, &partitionDef); err != nil {
return nil, err
}
}
// If unable to load from index bucket - attempt to initialize based on cbgt partitions
if partitionDef == nil {
var manager *cbgt.Manager
if k.context != nil {
manager = k.context.BucketSpec.CbgtContext.Manager
} else {
return nil, errors.New("Unable to determine partition map for index - not found in index, and no database context")
}
if manager == nil {
return nil, errors.New("Unable to determine partition map for index - not found in index, and no CBGT manager")
}
_, planPIndexesByName, _ := manager.GetPlanPIndexes(true)
indexName := k.context.GetCBGTIndexNameForBucket(k.context.Bucket)
pindexes := planPIndexesByName[indexName]
for index, pIndex := range pindexes {
vbStrings := strings.Split(pIndex.SourcePartitions, ",")
// convert string vbNos to uint16
vbNos := make([]uint16, len(vbStrings))
for i := 0; i < len(vbStrings); i++ {
vbNumber, err := strconv.ParseUint(vbStrings[i], 10, 16)
if err != nil {
base.LogFatal("Error creating index partition definition - unable to parse vbucket number %s as integer:%v", vbStrings[i], err)
}
vbNos[i] = uint16(vbNumber)
}
entry := base.PartitionStorage{
Index: uint16(index),
Uuid: pIndex.UUID,
VbNos: vbNos,
}
partitionDef = append(partitionDef, entry)
}
// Persist to bucket
value, err = json.Marshal(partitionDef)
if err != nil {
return nil, err
}
k.reader.indexReadBucket.SetRaw(base.KIndexPartitionKey, 0, value)
}
// Create k.indexPartitions based on partitionDef
k.indexPartitions = base.NewIndexPartitions(partitionDef)
return k.indexPartitions, nil
}
// InitRESTRouter initializes a mux.Router with REST API routes with
// extra option.
func InitRESTRouterEx(r *mux.Router, versionMain string,
mgr *cbgt.Manager, staticDir, staticETag string,
mr *cbgt.MsgRing,
assetDir func(name string) ([]string, error),
asset func(name string) ([]byte, error),
options map[string]interface{}) (
*mux.Router, map[string]RESTMeta, error) {
var authHandler func(http.Handler) http.Handler
mapRESTPathStats := map[string]*RESTPathStats{} // Keyed by path spec.
if options != nil {
if v, ok := options["auth"]; ok {
authHandler, ok = v.(func(http.Handler) http.Handler)
if !ok {
return nil, nil, fmt.Errorf("rest: auth function invalid")
}
}
if v, ok := options["mapRESTPathStats"]; ok {
mapRESTPathStats, ok = v.(map[string]*RESTPathStats)
if !ok {
return nil, nil, fmt.Errorf("rest: mapRESTPathStats invalid")
}
}
}
prefix := mgr.Options()["urlPrefix"]
PIndexTypesInitRouter(r, "manager.before", mgr)
meta := map[string]RESTMeta{}
handle := func(path string, method string, h http.Handler,
opts map[string]string) {
opts["_path"] = path
if a, ok := h.(RESTOpts); ok {
a.RESTOpts(opts)
}
prefixPath := prefix + path
restMeta := RESTMeta{prefixPath, method, opts}
meta[prefixPath+" "+RESTMethodOrds[method]+method] = restMeta
h = &HandlerWithRESTMeta{
h: h,
RESTMeta: &restMeta,
pathStats: mapRESTPathStats[path],
focusName: PathFocusName(path),
}
if authHandler != nil {
h = authHandler(h)
}
r.Handle(prefixPath, h).Methods(method).Name(prefixPath)
}
handle("/api/index", "GET", NewListIndexHandler(mgr),
map[string]string{
"_category": "Indexing|Index definition",
"_about": `Returns all index definitions as JSON.`,
"version introduced": "0.0.1",
})
handle("/api/index/{indexName}", "PUT", NewCreateIndexHandler(mgr),
map[string]string{
"_category": "Indexing|Index definition",
"_about": `Creates/updates an index definition.`,
"version introduced": "0.0.1",
})
handle("/api/index/{indexName}", "DELETE", NewDeleteIndexHandler(mgr),
map[string]string{
"_category": "Indexing|Index definition",
"_about": `Deletes an index definition.`,
"version introduced": "0.0.1",
})
handle("/api/index/{indexName}", "GET", NewGetIndexHandler(mgr),
map[string]string{
"_category": "Indexing|Index definition",
"_about": `Returns the definition of an index as JSON.`,
"version introduced": "0.0.1",
})
if mgr == nil || mgr.TagsMap() == nil || mgr.TagsMap()["queryer"] {
handle("/api/index/{indexName}/count", "GET",
NewCountHandler(mgr),
map[string]string{
"_category": "Indexing|Index querying",
"_about": `Returns the count of indexed documents.`,
"version introduced": "0.0.1",
})
handle("/api/index/{indexName}/query", "POST",
NewQueryHandler(mgr,
mapRESTPathStats["/api/index/{indexName}/query"]),
map[string]string{
"_category": "Indexing|Index querying",
"_about": `Queries an index.`,
"version introduced": "0.2.0",
})
}
handle("/api/index/{indexName}/planFreezeControl/{op}", "POST",
NewIndexControlHandler(mgr, "planFreeze", map[string]bool{
"freeze": true,
"unfreeze": true,
}),
map[string]string{
"_category": "Indexing|Index management",
"_about": `Freeze the assignment of index partitions to nodes.`,
"param: op": "required, string, URL path parameter\n\n" +
`Allowed values for op are "freeze" or "unfreeze".`,
"version introduced": "0.0.1",
})
handle("/api/index/{indexName}/ingestControl/{op}", "POST",
NewIndexControlHandler(mgr, "write", map[string]bool{
"pause": true,
"resume": true,
}),
map[string]string{
"_category": "Indexing|Index management",
"_about": `Pause index updates and maintenance (no more
ingesting document mutations).`,
"param: op": "required, string, URL path parameter\n\n" +
`Allowed values for op are "pause" or "resume".`,
"version introduced": "0.0.1",
})
handle("/api/index/{indexName}/queryControl/{op}", "POST",
NewIndexControlHandler(mgr, "read", map[string]bool{
"allow": true,
"disallow": true,
}),
map[string]string{
"_category": "Indexing|Index management",
"_about": `Disallow queries on an index.`,
"param: op": "required, string, URL path parameter\n\n" +
`Allowed values for op are "allow" or "disallow".`,
"version introduced": "0.0.1",
})
if mgr == nil || mgr.TagsMap() == nil || mgr.TagsMap()["pindex"] {
handle("/api/pindex", "GET",
NewListPIndexHandler(mgr),
map[string]string{
"_category": "x/Advanced|x/Index partition definition",
"version introduced": "0.0.1",
})
handle("/api/pindex/{pindexName}", "GET",
NewGetPIndexHandler(mgr),
map[string]string{
"_category": "x/Advanced|x/Index partition definition",
"version introduced": "0.0.1",
})
handle("/api/pindex/{pindexName}/count", "GET",
NewCountPIndexHandler(mgr),
map[string]string{
"_category": "x/Advanced|x/Index partition querying",
"version introduced": "0.0.1",
})
handle("/api/pindex/{pindexName}/query", "POST",
NewQueryPIndexHandler(mgr),
map[string]string{
"_category": "x/Advanced|x/Index partition querying",
"version introduced": "0.2.0",
})
}
handle("/api/managerOptions", "PUT", NewManagerOptions(mgr),
map[string]string{
"_category": "Node|Node configuration",
"_about": "Set the options for the manager",
"version introduced": "4.2.0",
})
handle("/api/cfg", "GET", NewCfgGetHandler(mgr),
map[string]string{
"_category": "Node|Node configuration",
"_about": `Returns the node's current view
of the cluster's configuration as JSON.`,
"version introduced": "0.0.1",
})
handle("/api/cfgRefresh", "POST", NewCfgRefreshHandler(mgr),
map[string]string{
"_category": "Node|Node configuration",
"_about": `Requests the node to refresh its configuration
from the configuration provider.`,
"version introduced": "0.0.1",
})
handle("/api/log", "GET", NewLogGetHandler(mgr, mr),
map[string]string{
"_category": "Node|Node diagnostics",
"_about": `Returns recent log messages
and key events for the node as JSON.`,
"version introduced": "0.0.1",
})
handle("/api/manager", "GET", NewManagerHandler(mgr),
map[string]string{
"_category": "Node|Node configuration",
"_about": `Returns runtime config information about this node.`,
"version introduced": "0.4.0",
})
handle("/api/managerKick", "POST", NewManagerKickHandler(mgr),
map[string]string{
"_category": "Node|Node configuration",
"_about": `Forces the node to replan resource assignments
(by running the planner, if enabled) and to update
its runtime state to reflect the latest plan
(by running the janitor, if enabled).`,
"version introduced": "0.0.1",
})
handle("/api/managerMeta", "GET", NewManagerMetaHandler(mgr, meta),
map[string]string{
"_category": "Node|Node configuration",
"_about": `Returns information on the node's capabilities,
including available indexing and storage options as JSON,
and is intended to help management tools and web UI's
to be more dynamically metadata driven.`,
"version introduced": "0.0.1",
})
handle("/api/runtime", "GET",
NewRuntimeGetHandler(versionMain, mgr),
map[string]string{
"_category": "Node|Node diagnostics",
"_about": `Returns information on the node's software,
such as version strings and slow-changing
runtime settings as JSON.`,
"version introduced": "0.0.1",
})
handle("/api/runtime/args", "GET",
http.HandlerFunc(RESTGetRuntimeArgs),
map[string]string{
"_category": "Node|Node diagnostics",
"_about": `Returns information on the node's command-line,
parameters, environment variables and
O/S process values as JSON.`,
"version introduced": "0.0.1",
})
handle("/api/diag", "GET",
NewDiagGetHandler(versionMain, mgr, mr, assetDir, asset),
map[string]string{
"_category": "Node|Node diagnostics",
"_about": `Returns full set of diagnostic information
from the node in one shot as JSON. That is, the
/api/diag response will be the union of the responses
from the other REST API diagnostic and monitoring
endpoints from the node, and is intended to make
production support easier.`,
"version introduced": "0.0.1",
})
handle("/api/ping", "GET", &NoopHandler{},
map[string]string{
"_category": "Node|Node diagnostics",
"_about": `Returns an empty body as a quick aliveness check.`,
"version introduced": "5.0.0",
})
handle("/api/runtime/gc", "POST",
http.HandlerFunc(RESTPostRuntimeGC),
map[string]string{
"_category": "Node|Node management",
"_about": `Requests the node to perform a GC.`,
"version introduced": "0.0.1",
})
handle("/api/runtime/profile/cpu", "POST",
http.HandlerFunc(RESTProfileCPU),
map[string]string{
"_category": "Node|Node diagnostics",
"_about": `Requests the node to capture local
cpu usage profiling information.`,
"version introduced": "0.0.1",
})
handle("/api/runtime/profile/memory", "POST",
http.HandlerFunc(RESTProfileMemory),
map[string]string{
"_category": "Node|Node diagnostics",
"_about": `Requests the node to capture lcoal
memory usage profiling information.`,
"version introduced": "0.0.1",
})
handle("/api/runtime/stats", "GET",
http.HandlerFunc(RESTGetRuntimeStats),
map[string]string{
"_category": "Node|Node monitoring",
"_about": `Returns information on the node's
low-level runtime stats as JSON.`,
"version introduced": "0.0.1",
})
handle("/api/runtime/statsMem", "GET",
http.HandlerFunc(RESTGetRuntimeStatsMem),
map[string]string{
"_category": "Node|Node monitoring",
"_about": `Returns information on the node's
low-level GC and memory related runtime stats as JSON.`,
"version introduced": "0.0.1",
})
handle("/api/stats", "GET", NewStatsHandler(mgr),
map[string]string{
"_category": "Indexing|Index monitoring",
"_about": `Returns indexing and data related metrics,
timings and counters from the node as JSON.`,
"version introduced": "0.0.1",
})
// TODO: If we ever implement cluster-wide index stats, we should
// have it under /api/index/{indexName}/stats GET endpoint.
//
handle("/api/stats/index/{indexName}", "GET", NewStatsHandler(mgr),
map[string]string{
"_category": "Indexing|Index monitoring",
"_about": `Returns metrics, timings and counters
for a single index from the node as JSON.`,
"version introduced": "0.0.1",
})
handle("/api/stats/sourceStats/{indexName}", "GET",
NewSourceStatsHandler(mgr),
map[string]string{
"_category": "Indexing|Index monitoring",
"_about": `Returns data source specific stats
for an index as JSON.`,
"version introduced": "4.2.0",
})
handle("/api/stats/sourcePartitionSeqs/{indexName}", "GET",
NewSourcePartitionSeqsHandler(mgr),
map[string]string{
"_category": "Indexing|Index monitoring",
"_about": `Returns data source partiton seqs
for an index as JSON.`,
"version introduced": "4.2.0",
})
PIndexTypesInitRouter(r, "manager.after", mgr)
return r, meta, nil
}
// The indexName/indexUUID is for a user-defined index alias.
//
// TODO: One day support user-defined aliases for non-bleve indexes.
func bleveIndexAliasForUserIndexAlias(mgr *cbgt.Manager,
indexName, indexUUID string, ensureCanRead bool,
consistencyParams *cbgt.ConsistencyParams,
cancelCh <-chan bool) (
bleve.IndexAlias, error) {
alias := bleve.NewIndexAlias()
indexDefs, _, err := cbgt.CfgGetIndexDefs(mgr.Cfg())
if err != nil {
return nil, fmt.Errorf("alias: could not get indexDefs,"+
" indexName: %s, err: %v", indexName, err)
}
num := 0
var fillAlias func(aliasName, aliasUUID string) error
fillAlias = func(aliasName, aliasUUID string) error {
aliasDef := indexDefs.IndexDefs[aliasName]
if aliasDef == nil {
return fmt.Errorf("alias: could not get aliasDef,"+
" aliasName: %s, indexName: %s",
aliasName, indexName)
}
if aliasDef.Type != "alias" {
return fmt.Errorf("alias: not alias type: %s,"+
" aliasName: %s, indexName: %s",
aliasDef.Type, aliasName, indexName)
}
if aliasUUID != "" &&
aliasUUID != aliasDef.UUID {
return fmt.Errorf("alias: mismatched aliasUUID: %s,"+
" aliasDef.UUID: %s, aliasName: %s, indexName: %s",
aliasUUID, aliasDef.UUID, aliasName, indexName)
}
params := AliasParams{}
err := json.Unmarshal([]byte(aliasDef.Params), ¶ms)
if err != nil {
return fmt.Errorf("alias: could not parse aliasDef.Params: %s,"+
" aliasName: %s, indexName: %s",
aliasDef.Params, aliasName, indexName)
}
for targetName, targetSpec := range params.Targets {
if num > maxAliasTargets {
return fmt.Errorf("alias: too many alias targets,"+
" perhaps there's a cycle,"+
" aliasName: %s, indexName: %s",
aliasName, indexName)
}
targetDef := indexDefs.IndexDefs[targetName]
if targetDef == nil {
return fmt.Errorf("alias: the alias depends upon"+
" a target index that does not exist,"+
" targetName: %q, aliasName: %q",
targetName, aliasName)
}
if targetSpec.IndexUUID != "" &&
targetSpec.IndexUUID != targetDef.UUID {
return fmt.Errorf("alias: mismatched targetSpec.UUID: %s,"+
" targetDef.UUID: %s, targetName: %s,"+
" aliasName: %s, indexName: %s",
targetSpec.IndexUUID, targetDef.UUID, targetName,
aliasName, indexName)
}
// TODO: Convert to registered callbacks instead of if-else-if.
if targetDef.Type == "alias" {
err = fillAlias(targetName, targetSpec.IndexUUID)
if err != nil {
return err
}
} else if strings.HasPrefix(targetDef.Type, "bleve") {
subAlias, err := bleveIndexAlias(mgr, targetName,
targetSpec.IndexUUID, ensureCanRead,
consistencyParams, cancelCh)
if err != nil {
return err
}
alias.Add(subAlias)
num += 1
} else {
return fmt.Errorf("alias: unsupported target type: %s,"+
" targetName: %s, aliasName: %s, indexName: %s",
targetDef.Type, targetName, aliasName, indexName)
}
}
return nil
}
err = fillAlias(indexName, indexUUID)
if err != nil {
return nil, err
}
return alias, nil
}
func testCreateIndex(t *testing.T,
mgr *cbgt.Manager,
indexName string,
params map[string]string,
waitUntilEmptyCfgEventsIndexDefs func()) {
sourceType := "primary"
if params["sourceType"] != "" {
sourceType = params["sourceType"]
}
if params[indexName+".sourceType"] != "" {
sourceType = params[indexName+".sourceType"]
}
sourceName := "default"
if params["sourceName"] != "" {
sourceName = params["sourceName"]
}
if params[indexName+".sourceName"] != "" {
sourceName = params[indexName+".sourceName"]
}
sourceUUID := ""
if params["sourceUUID"] != "" {
sourceUUID = params["sourceUUID"]
}
if params[indexName+".sourceUUID"] != "" {
sourceUUID = params[indexName+".sourceUUID"]
}
sourceParams := `{"numPartitions":4}`
if params["sourceParams"] != "" {
sourceParams = params["sourceParams"]
}
if params[indexName+".sourceParams"] != "" {
sourceParams = params[indexName+".sourceParams"]
}
indexType := "blackhole"
if params["indexType"] != "" {
indexType = params["indexType"]
}
if params[indexName+".indexType"] != "" {
indexType = params[indexName+".indexType"]
}
indexParams := ""
if params["indexParams"] != "" {
indexParams = params["indexParams"]
}
if params[indexName+".indexParams"] != "" {
indexParams = params[indexName+".indexParams"]
}
prevIndexUUID := ""
if params["prevIndexUUID"] != "" {
prevIndexUUID = params["prevIndexUUID"]
}
if params[indexName+".prevIndexUUID"] != "" {
prevIndexUUID = params[indexName+".prevIndexUUID"]
}
planParams := cbgt.PlanParams{
MaxPartitionsPerPIndex: 1,
}
waitUntilEmptyCfgEventsIndexDefs()
err := mgr.CreateIndex(
sourceType, sourceName, sourceUUID, sourceParams,
indexType, indexName, indexParams,
planParams,
prevIndexUUID)
if err != nil {
t.Errorf("expected no err, got: %#v", err)
}
waitUntilEmptyCfgEventsIndexDefs()
}
// InitRESTRouter initializes a mux.Router with REST API
// routes.
func InitRESTRouter(r *mux.Router, versionMain string,
mgr *cbgt.Manager, staticDir, staticETag string,
mr *cbgt.MsgRing,
assetDir func(name string) ([]string, error),
asset func(name string) ([]byte, error)) (
*mux.Router, map[string]RESTMeta, error) {
PIndexTypesInitRouter(r, "manager.before")
meta := map[string]RESTMeta{}
handle := func(path string, method string, h http.Handler,
opts map[string]string) {
if a, ok := h.(RESTOpts); ok {
a.RESTOpts(opts)
}
meta[path+" "+RESTMethodOrds[method]+method] =
RESTMeta{path, method, opts}
r.Handle(path, h).Methods(method)
}
handleFunc := func(path string, method string, h http.HandlerFunc,
opts map[string]string) {
meta[path+" "+RESTMethodOrds[method]+method] =
RESTMeta{path, method, opts}
r.HandleFunc(path, h).Methods(method)
}
handle("/api/index", "GET", NewListIndexHandler(mgr),
map[string]string{
"_category": "Indexing|Index definition",
"_about": `Returns all index definitions as JSON.`,
"version introduced": "0.0.1",
})
handle("/api/index/{indexName}", "PUT", NewCreateIndexHandler(mgr),
map[string]string{
"_category": "Indexing|Index definition",
"_about": `Creates/updates an index definition.`,
"version introduced": "0.0.1",
})
handle("/api/index/{indexName}", "DELETE", NewDeleteIndexHandler(mgr),
map[string]string{
"_category": "Indexing|Index definition",
"_about": `Deletes an index definition.`,
"version introduced": "0.0.1",
})
handle("/api/index/{indexName}", "GET", NewGetIndexHandler(mgr),
map[string]string{
"_category": "Indexing|Index definition",
"_about": `Returns the definition of an index as JSON.`,
"version introduced": "0.0.1",
})
if mgr == nil || mgr.TagsMap() == nil || mgr.TagsMap()["queryer"] {
handle("/api/index/{indexName}/count", "GET",
NewCountHandler(mgr),
map[string]string{
"_category": "Indexing|Index querying",
"_about": `Returns the count of indexed documents.`,
"version introduced": "0.0.1",
})
handle("/api/index/{indexName}/query", "POST",
NewQueryHandler(mgr),
map[string]string{
"_category": "Indexing|Index querying",
"_about": `Queries an index.`,
"version introduced": "0.2.0",
})
}
handle("/api/index/{indexName}/planFreezeControl/{op}", "POST",
NewIndexControlHandler(mgr, "planFreeze", map[string]bool{
"freeze": true,
"unfreeze": true,
}),
map[string]string{
"_category": "Indexing|Index management",
"_about": `Freeze the assignment of index partitions to nodes.`,
"param: op": "required, string, URL path parameter\n\n" +
`Allowed values for op are "freeze" or "unfreeze".`,
"version introduced": "0.0.1",
})
handle("/api/index/{indexName}/ingestControl/{op}", "POST",
NewIndexControlHandler(mgr, "write", map[string]bool{
"pause": true,
"resume": true,
}),
map[string]string{
"_category": "Indexing|Index management",
"_about": `Pause index updates and maintenance (no more
ingesting document mutations).`,
"param: op": "required, string, URL path parameter\n\n" +
`Allowed values for op are "pause" or "resume".`,
"version introduced": "0.0.1",
})
handle("/api/index/{indexName}/queryControl/{op}", "POST",
NewIndexControlHandler(mgr, "read", map[string]bool{
"allow": true,
"disallow": true,
}),
map[string]string{
"_category": "Indexing|Index management",
"_about": `Disallow queries on an index.`,
"param: op": "required, string, URL path parameter\n\n" +
`Allowed values for op are "allow" or "disallow".`,
"version introduced": "0.0.1",
})
if mgr == nil || mgr.TagsMap() == nil || mgr.TagsMap()["pindex"] {
handle("/api/pindex", "GET",
NewListPIndexHandler(mgr),
map[string]string{
"_category": "x/Advanced|x/Index partition definition",
"version introduced": "0.0.1",
})
handle("/api/pindex/{pindexName}", "GET",
NewGetPIndexHandler(mgr),
map[string]string{
"_category": "x/Advanced|x/Index partition definition",
"version introduced": "0.0.1",
})
handle("/api/pindex/{pindexName}/count", "GET",
NewCountPIndexHandler(mgr),
map[string]string{
"_category": "x/Advanced|x/Index partition querying",
"version introduced": "0.0.1",
})
handle("/api/pindex/{pindexName}/query", "POST",
NewQueryPIndexHandler(mgr),
map[string]string{
"_category": "x/Advanced|x/Index partition querying",
"version introduced": "0.2.0",
})
}
handle("/api/cfg", "GET", NewCfgGetHandler(mgr),
map[string]string{
"_category": "Node|Node configuration",
"_about": `Returns the node's current view
of the cluster's configuration as JSON.`,
"version introduced": "0.0.1",
})
handle("/api/cfgRefresh", "POST", NewCfgRefreshHandler(mgr),
map[string]string{
"_category": "Node|Node configuration",
"_about": `Requests the node to refresh its configuration
from the configuration provider.`,
"version introduced": "0.0.1",
})
handle("/api/log", "GET", NewLogGetHandler(mgr, mr),
map[string]string{
"_category": "Node|Node diagnostics",
"_about": `Returns recent log messages
and key events for the node as JSON.`,
"version introduced": "0.0.1",
})
handle("/api/managerKick", "POST", NewManagerKickHandler(mgr),
map[string]string{
"_category": "Node|Node configuration",
"_about": `Forces the node to replan resource assignments
(by running the planner, if enabled) and to update
its runtime state to reflect the latest plan
(by running the janitor, if enabled).`,
"version introduced": "0.0.1",
})
handle("/api/managerMeta", "GET", NewManagerMetaHandler(mgr, meta),
map[string]string{
"_category": "Node|Node configuration",
"_about": `Returns information on the node's capabilities,
including available indexing and storage options as JSON,
and is intended to help management tools and web UI's
to be more dynamically metadata driven.`,
"version introduced": "0.0.1",
})
handle("/api/runtime", "GET",
NewRuntimeGetHandler(versionMain, mgr),
map[string]string{
"_category": "Node|Node diagnostics",
"_about": `Returns information on the node's software,
such as version strings and slow-changing
runtime settings as JSON.`,
"version introduced": "0.0.1",
})
handleFunc("/api/runtime/args", "GET",
RESTGetRuntimeArgs, map[string]string{
"_category": "Node|Node diagnostics",
"_about": `Returns information on the node's command-line,
parameters, environment variables and
O/S process values as JSON.`,
"version introduced": "0.0.1",
})
handle("/api/diag", "GET",
NewDiagGetHandler(versionMain, mgr, mr, assetDir, asset),
map[string]string{
"_category": "Node|Node diagnostics",
"_about": `Returns full set of diagnostic information
from the node in one shot as JSON. That is, the
/api/diag response will be the union of the responses
from the other REST API diagnostic and monitoring
endpoints from the node, and is intended to make
production support easier.`,
"version introduced": "0.0.1",
})
handleFunc("/api/runtime/gc", "POST",
RESTPostRuntimeGC, map[string]string{
"_category": "Node|Node management",
"_about": `Requests the node to perform a GC.`,
"version introduced": "0.0.1",
})
handleFunc("/api/runtime/profile/cpu", "POST",
RESTProfileCPU, map[string]string{
"_category": "Node|Node diagnostics",
"_about": `Requests the node to capture local
cpu usage profiling information.`,
"version introduced": "0.0.1",
})
handleFunc("/api/runtime/profile/memory", "POST",
RESTProfileMemory, map[string]string{
"_category": "Node|Node diagnostics",
"_about": `Requests the node to capture lcoal
memory usage profiling information.`,
"version introduced": "0.0.1",
})
handleFunc("/api/runtime/stats", "GET",
RESTGetRuntimeStats, map[string]string{
"_category": "Node|Node monitoring",
"_about": `Returns information on the node's
low-level runtime stats as JSON.`,
"version introduced": "0.0.1",
})
handleFunc("/api/runtime/statsMem", "GET",
RESTGetRuntimeStatsMem, map[string]string{
"_category": "Node|Node monitoring",
"_about": `Returns information on the node's
low-level GC and memory related runtime stats as JSON.`,
"version introduced": "0.0.1",
})
handle("/api/stats", "GET", NewStatsHandler(mgr),
map[string]string{
"_category": "Indexing|Index monitoring",
"_about": `Returns indexing and data related metrics,
timings and counters from the node as JSON.`,
"version introduced": "0.0.1",
})
// TODO: If we ever implement cluster-wide index stats, we should
// have it under /api/index/{indexName}/stats GET endpoint.
//
handle("/api/stats/index/{indexName}", "GET", NewStatsHandler(mgr),
map[string]string{
"_category": "Indexing|Index monitoring",
"_about": `Returns metrics, timings and counters
for a single index from the node as JSON.`,
"version introduced": "0.0.1",
})
PIndexTypesInitRouter(r, "manager.after")
return r, meta, nil
}
func initNsServerCaching(mgr *cbgt.Manager) {
runSourcePartitionSeqsOnce.Do(func() {
go RunSourcePartitionSeqs(mgr.Options(), nil)
go RunRecentInfoCache(mgr)
})
}
func RunRecentInfoCache(mgr *cbgt.Manager) {
cfg := mgr.Cfg()
cfgChangedCh := make(chan struct{}, 10)
go func() { // Debounce cfg events to feed into the cfgChangedCh.
ech := make(chan cbgt.CfgEvent)
cfg.Subscribe(cbgt.PLAN_PINDEXES_KEY, ech)
for {
<-ech // First, wait for a cfg event.
debounceTimeCh := time.After(500 * time.Millisecond)
DEBOUNCE_LOOP:
for {
select {
case <-ech:
// NO-OP when there are more, spammy cfg events.
case <-debounceTimeCh:
break DEBOUNCE_LOOP
}
}
cfgChangedCh <- struct{}{}
}
}()
tickCh := time.Tick(1 * time.Minute)
for {
var nodeDefs *cbgt.NodeDefs
var planPIndexes *cbgt.PlanPIndexes
indexDefs, indexDefsMap, err := mgr.GetIndexDefs(false)
if err == nil {
nodeDefs, _, err = cbgt.CfgGetNodeDefs(cfg, cbgt.NODE_DEFS_WANTED)
if err == nil {
planPIndexes, _, err = cbgt.CfgGetPlanPIndexes(cfg)
}
}
rd := &recentInfo{
indexDefs: indexDefs,
indexDefsMap: indexDefsMap,
nodeDefs: nodeDefs,
planPIndexes: planPIndexes,
err: err,
}
runtime.ReadMemStats(&rd.memStats)
REUSE_CACHE:
for {
select {
case <-cfgChangedCh:
break REUSE_CACHE
case <-tickCh:
break REUSE_CACHE
case recentInfoCh <- rd:
if rd.err != nil {
break REUSE_CACHE
}
}
}
}
}
func QueryBlevePIndexImpl(mgr *cbgt.Manager, indexName, indexUUID string,
req []byte, res io.Writer) error {
queryCtlParams := cbgt.QueryCtlParams{
Ctl: cbgt.QueryCtl{
Timeout: cbgt.QUERY_CTL_DEFAULT_TIMEOUT_MS,
},
}
err := json.Unmarshal(req, &queryCtlParams)
if err != nil {
return fmt.Errorf("bleve: QueryBlevePIndexImpl"+
" parsing queryCtlParams, req: %s, err: %v", req, err)
}
searchRequest := &bleve.SearchRequest{}
err = json.Unmarshal(req, searchRequest)
if err != nil {
return fmt.Errorf("bleve: QueryBlevePIndexImpl"+
" parsing searchRequest, req: %s, err: %v", req, err)
}
err = searchRequest.Query.Validate()
if err != nil {
return err
}
v, exists := mgr.Options()["bleveMaxResultWindow"]
if exists {
bleveMaxResultWindow, err := strconv.Atoi(v)
if err != nil {
return err
}
if searchRequest.From+searchRequest.Size > bleveMaxResultWindow {
return fmt.Errorf("bleve: bleveMaxResultWindow exceeded,"+
" from: %d, size: %d, bleveMaxResultWindow: %d",
searchRequest.From, searchRequest.Size, bleveMaxResultWindow)
}
}
cancelCh := cbgt.TimeoutCancelChan(queryCtlParams.Ctl.Timeout)
alias, err := bleveIndexAlias(mgr, indexName, indexUUID, true,
queryCtlParams.Ctl.Consistency, cancelCh)
if err != nil {
return err
}
doneCh := make(chan struct{})
var searchResult *bleve.SearchResult
go func() {
searchResult, err = alias.Search(searchRequest)
close(doneCh)
}()
select {
case <-cancelCh:
err = fmt.Errorf("pindex_bleve: query timeout")
case <-doneCh:
if searchResult != nil {
rest.MustEncode(res, searchResult)
}
}
return err
}
