func QueryBlevePIndexImpl(mgr *cbgt.Manager, indexName, indexUUID string,
req []byte, res io.Writer) error {
queryCtlParams := cbgt.QueryCtlParams{
Ctl: cbgt.QueryCtl{
Timeout: cbgt.QUERY_CTL_DEFAULT_TIMEOUT_MS,
},
}
err := json.Unmarshal(req, &queryCtlParams)
if err != nil {
return fmt.Errorf("bleve: QueryBlevePIndexImpl"+
" parsing queryCtlParams, req: %s, err: %v", req, err)
}
searchRequest := &bleve.SearchRequest{}
err = json.Unmarshal(req, searchRequest)
if err != nil {
return fmt.Errorf("bleve: QueryBlevePIndexImpl"+
" parsing searchRequest, req: %s, err: %v", req, err)
}
err = searchRequest.Query.Validate()
if err != nil {
return err
}
cancelCh := cbgt.TimeoutCancelChan(queryCtlParams.Ctl.Timeout)
alias, err := bleveIndexAlias(mgr, indexName, indexUUID, true,
queryCtlParams.Ctl.Consistency, cancelCh)
if err != nil {
return err
}
doneCh := make(chan struct{})
var searchResult *bleve.SearchResult
go func() {
searchResult, err = alias.Search(searchRequest)
close(doneCh)
}()
select {
case <-cancelCh:
err = fmt.Errorf("pindex_bleve: query timeout")
case <-doneCh:
if searchResult != nil {
rest.MustEncode(res, searchResult)
}
}
return err
}
func (h *NsStatusHandler) ServeHTTP(
w http.ResponseWriter, req *http.Request) {
initNsServerCaching(h.mgr)
rd := <-recentInfoCh
if rd.err != nil {
rest.ShowError(w, req, fmt.Sprintf("could not retrieve defs: %v", rd.err), 500)
return
}
indexDefsMap := rd.indexDefsMap
nodeDefs := rd.nodeDefs
planPIndexes := rd.planPIndexes
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Write(cbgt.JsonOpenBrace)
w.Write(statsNamePrefix)
w.Write([]byte("status"))
w.Write(statsNameSuffix)
w.Write([]byte("["))
indexDefNames := make(sort.StringSlice, 0, len(indexDefsMap))
for indexDefName := range indexDefsMap {
indexDefNames = append(indexDefNames, indexDefName)
}
sort.Sort(indexDefNames)
for i, indexDefName := range indexDefNames {
indexDef := indexDefsMap[indexDefName]
if i > 0 {
w.Write(cbgt.JsonComma)
}
rest.MustEncode(w, struct {
Hosts []string `json:"hosts"`
Bucket string `json:"bucket"`
Name string `json:"name"`
}{
Bucket: indexDef.SourceName,
Name: indexDefName,
Hosts: NsHostsForIndex(indexDefName, planPIndexes, nodeDefs),
})
}
w.Write([]byte("],"))
w.Write(statsNamePrefix)
w.Write([]byte("code"))
w.Write(statsNameSuffix)
w.Write([]byte("\"success\""))
w.Write(cbgt.JsonCloseBrace)
}
func (t *BleveDest) Query(pindex *cbgt.PIndex, req []byte, res io.Writer,
cancelCh <-chan bool) error {
queryCtlParams := cbgt.QueryCtlParams{
Ctl: cbgt.QueryCtl{
Timeout: cbgt.QUERY_CTL_DEFAULT_TIMEOUT_MS,
},
}
err := json.Unmarshal(req, &queryCtlParams)
if err != nil {
return fmt.Errorf("bleve: BleveDest.Query"+
" parsing queryCtlParams, req: %s, err: %v", req, err)
}
searchRequest := &bleve.SearchRequest{}
err = json.Unmarshal(req, searchRequest)
if err != nil {
return fmt.Errorf("bleve: BleveDest.Query"+
" parsing searchRequest, req: %s, err: %v", req, err)
}
err = cbgt.ConsistencyWaitPIndex(pindex, t,
queryCtlParams.Ctl.Consistency, cancelCh)
if err != nil {
return err
}
err = searchRequest.Query.Validate()
if err != nil {
return err
}
t.m.Lock()
bindex := t.bindex
t.m.Unlock()
if bindex == nil {
return fmt.Errorf("bleve: Query, bindex already closed")
}
searchResponse, err := bindex.Search(searchRequest)
if err != nil {
return err
}
rest.MustEncode(res, searchResponse)
return nil
}
func QueryAlias(mgr *cbgt.Manager, indexName, indexUUID string,
req []byte, res io.Writer) error {
queryCtlParams := cbgt.QueryCtlParams{
Ctl: cbgt.QueryCtl{
Timeout: cbgt.QUERY_CTL_DEFAULT_TIMEOUT_MS,
},
}
err := json.Unmarshal(req, &queryCtlParams)
if err != nil {
return fmt.Errorf("alias: QueryAlias"+
" parsing queryCtlParams, req: %s, err: %v", req, err)
}
searchRequest := &bleve.SearchRequest{}
err = json.Unmarshal(req, searchRequest)
if err != nil {
return fmt.Errorf("alias: QueryAlias"+
" parsing searchRequest, req: %s, err: %v", req, err)
}
if srqv, ok := searchRequest.Query.(query.ValidatableQuery); ok {
err = srqv.Validate()
if err != nil {
return err
}
}
cancelCh := cbgt.TimeoutCancelChan(queryCtlParams.Ctl.Timeout)
alias, err := bleveIndexAliasForUserIndexAlias(mgr,
indexName, indexUUID, true,
queryCtlParams.Ctl.Consistency, cancelCh, true)
if err != nil {
return err
}
searchResponse, err := alias.Search(searchRequest)
if err != nil {
return err
}
rest.MustEncode(res, searchResponse)
return nil
}
func CheckAPIVersion(w http.ResponseWriter, req *http.Request) (err error) {
var version = API_MAX_VERSION
if req.Header != nil && len(req.Header["Accept"]) > 0 {
version, err = HandleAPIVersion(req.Header["Accept"][0])
if err != nil {
w.WriteHeader(406)
versionList := []string{
WithJSONVersion(API_MAX_VERSION),
WithJSONVersion(API_MIN_VERSION),
}
rest.MustEncode(w, versionList)
return err
}
}
w.Header().Set("Content-type", WithJSONVersion(version))
return nil
}
func (h *NsStatsHandler) ServeHTTP(
w http.ResponseWriter, req *http.Request) {
_, indexDefsMap, err := h.mgr.GetIndexDefs(false)
if err != nil {
rest.ShowError(w, req, "could not retrieve index defs", 500)
return
}
nsIndexStats := make(NSIndexStats, len(indexDefsMap))
for indexDefName, indexDef := range indexDefsMap {
nsIndexStats[indexDef.SourceName+":"+indexDefName] = NewIndexStat()
}
feeds, pindexes := h.mgr.CurrentMaps()
sourceName := ""
for _, pindex := range pindexes {
sourceName = pindex.SourceName
lindexName := pindex.SourceName + ":" + pindex.IndexName
nsIndexStat, ok := nsIndexStats[lindexName]
if ok {
// manually track a statistic representing
// the number of pindex in the index
oldValue, ok := nsIndexStat["num_pindexes"]
if ok {
switch oldValue := oldValue.(type) {
case float64:
oldValue += float64(1)
nsIndexStat["num_pindexes"] = oldValue
}
}
// automatically process all the pindex dest stats
err := addPindexStats(pindex, nsIndexStat)
if err != nil {
rest.ShowError(w, req, fmt.Sprintf("error processing PIndex stats: %v", err), 500)
return
}
}
}
for _, feed := range feeds {
lindexName := sourceName + ":" + feed.IndexName()
nsIndexStat, ok := nsIndexStats[lindexName]
if ok {
err := addFeedStats(feed, nsIndexStat)
// automatically process all the feed stats
if err != nil {
rest.ShowError(w, req, fmt.Sprintf("error processing Feed stats: %v", err), 500)
return
}
}
}
// FIXME hard-coded top-level stats
nsIndexStats[""] = make(map[string]interface{})
nsIndexStats[""]["num_connections"] = 0
nsIndexStats[""]["needs_restart"] = false
rest.MustEncode(w, nsIndexStats)
}
func (h *NsStatusHandler) ServeHTTP(
w http.ResponseWriter, req *http.Request) {
cfg := h.mgr.Cfg()
planPIndexes, _, err := cbgt.CfgGetPlanPIndexes(cfg)
if err != nil {
rest.ShowError(w, req, "could not retrieve plan pIndexes", 500)
return
}
nodesDefs, _, err := cbgt.CfgGetNodeDefs(cfg, cbgt.NODE_DEFS_WANTED)
if err != nil {
rest.ShowError(w, req, "could not retrieve node defs (wanted)", 500)
return
}
_, indexDefsMap, err := h.mgr.GetIndexDefs(false)
if err != nil {
rest.ShowError(w, req, "could not retrieve index defs", 500)
return
}
w.Write(cbgt.JsonOpenBrace)
w.Write(statsNamePrefix)
w.Write([]byte("status"))
w.Write(statsNameSuffix)
w.Write([]byte("["))
indexDefNames := make(sort.StringSlice, 0, len(indexDefsMap))
for indexDefName := range indexDefsMap {
indexDefNames = append(indexDefNames, indexDefName)
}
sort.Sort(indexDefNames)
for i, indexDefName := range indexDefNames {
indexDef := indexDefsMap[indexDefName]
if i > 0 {
w.Write(cbgt.JsonComma)
}
rest.MustEncode(w, struct {
Completion int `json:"completion"`
Hosts []string `json:"hosts"`
Status string `json:"status"`
Bucket string `json:"bucket"`
Name string `json:"name"`
}{
Bucket: indexDef.SourceName,
Name: indexDefName,
Hosts: HostsForIndex(indexDefName, planPIndexes, nodesDefs),
// FIXME hard-coded
Completion: 100,
Status: "Ready",
})
}
w.Write([]byte("],"))
w.Write(statsNamePrefix)
w.Write([]byte("code"))
w.Write(statsNameSuffix)
w.Write([]byte("\"success\""))
w.Write(cbgt.JsonCloseBrace)
}
func (h *NsStatsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
currentStatsCount := atomic.AddInt64(&h.statsCount, 1)
initNsServerCaching(h.mgr)
rd := <-recentInfoCh
if rd.err != nil {
rest.ShowError(w, req, fmt.Sprintf("could not retrieve defs: %v", rd.err), 500)
return
}
indexDefsMap := rd.indexDefsMap
planPIndexes := rd.planPIndexes
nodeUUID := h.mgr.UUID()
nsIndexStats := make(NSIndexStats, len(indexDefsMap))
indexNameToSourceName := map[string]string{}
indexNameToPlanPIndexes := map[string][]*cbgt.PlanPIndex{}
if planPIndexes != nil {
for _, planPIndex := range planPIndexes.PlanPIndexes {
// Only focus on the planPIndex entries for this node.
if planPIndex.Nodes[nodeUUID] != nil {
indexNameToPlanPIndexes[planPIndex.IndexName] =
append(indexNameToPlanPIndexes[planPIndex.IndexName], planPIndex)
}
}
}
indexQueryPathStats := MapRESTPathStats[RESTIndexQueryPath]
// Keyed by indexName, sub-key is source partition id.
indexNameToSourcePartitionSeqs := map[string]map[string]cbgt.UUIDSeq{}
// Keyed by indexName, sub-key is source partition id.
indexNameToDestPartitionSeqs := map[string]map[string]cbgt.UUIDSeq{}
for indexName, indexDef := range indexDefsMap {
nsIndexStat := NewIndexStat()
nsIndexStats[indexDef.SourceName+":"+indexName] = nsIndexStat
indexNameToSourceName[indexName] = indexDef.SourceName
focusStats := indexQueryPathStats.FocusStats(indexName)
if focusStats != nil {
totalQueries := atomic.LoadUint64(&focusStats.TotRequest)
nsIndexStat["total_queries"] = totalQueries
if totalQueries > 0 {
nsIndexStat["avg_queries_latency"] =
float64((atomic.LoadUint64(&focusStats.TotRequestTimeNS) /
totalQueries)) / 1000000.0 // Convert from nanosecs to millisecs.
}
nsIndexStat["total_request_time"] =
atomic.LoadUint64(&focusStats.TotRequestTimeNS)
nsIndexStat["total_queries_slow"] =
atomic.LoadUint64(&focusStats.TotRequestSlow)
nsIndexStat["total_queries_timeout"] =
atomic.LoadUint64(&focusStats.TotRequestTimeout)
nsIndexStat["total_queries_error"] =
atomic.LoadUint64(&focusStats.TotRequestErr)
nsIndexStat["total_bytes_query_results"] =
atomic.LoadUint64(&focusStats.TotResponseBytes)
nsIndexStat["num_pindexes_target"] =
uint64(len(indexNameToPlanPIndexes[indexName]))
}
feedType, exists := cbgt.FeedTypes[indexDef.SourceType]
if !exists || feedType == nil || feedType.PartitionSeqs == nil {
continue
}
partitionSeqs := GetSourcePartitionSeqs(SourceSpec{
SourceType: indexDef.SourceType,
SourceName: indexDef.SourceName,
SourceUUID: indexDef.SourceUUID,
SourceParams: indexDef.SourceParams,
Server: h.mgr.Server(),
})
if partitionSeqs != nil {
indexNameToSourcePartitionSeqs[indexName] = partitionSeqs
}
}
feeds, pindexes := h.mgr.CurrentMaps()
for _, pindex := range pindexes {
nsIndexName := pindex.SourceName + ":" + pindex.IndexName
nsIndexStat, ok := nsIndexStats[nsIndexName]
if ok {
// manually track num pindexes
oldValue, ok := nsIndexStat["num_pindexes_actual"]
if ok {
switch oldValue := oldValue.(type) {
case float64:
oldValue += float64(1)
nsIndexStat["num_pindexes_actual"] = oldValue
// TODO: Former name was num_pindexes, need to remove one day.
nsIndexStat["num_pindexes"] = oldValue
}
}
// automatically process all the pindex dest stats
err := addPIndexStats(pindex, nsIndexStat)
if err != nil {
rest.ShowError(w, req,
fmt.Sprintf("error processing PIndex stats: %v", err), 500)
return
}
dest := pindex.Dest
if dest != nil {
destForwarder, ok := dest.(*cbgt.DestForwarder)
if !ok {
continue
}
partitionSeqsProvider, ok :=
destForwarder.DestProvider.(PartitionSeqsProvider)
if !ok {
continue
}
partitionSeqs, err := partitionSeqsProvider.PartitionSeqs()
if err == nil {
m := indexNameToDestPartitionSeqs[pindex.IndexName]
if m == nil {
m = map[string]cbgt.UUIDSeq{}
indexNameToDestPartitionSeqs[pindex.IndexName] = m
}
for partitionId, uuidSeq := range partitionSeqs {
m[partitionId] = uuidSeq
}
}
}
}
}
for _, feed := range feeds {
sourceName := indexNameToSourceName[feed.IndexName()]
nsIndexName := sourceName + ":" + feed.IndexName()
nsIndexStat, ok := nsIndexStats[nsIndexName]
if ok {
// automatically process all the feed stats
err := addFeedStats(feed, nsIndexStat)
if err != nil {
rest.ShowError(w, req,
fmt.Sprintf("error processing Feed stats: %v", err), 500)
return
}
}
}
for indexName, indexDef := range indexDefsMap {
nsIndexStat, ok := nsIndexStats[indexDef.SourceName+":"+indexName]
if ok {
src := indexNameToSourcePartitionSeqs[indexName]
if src == nil {
continue
}
dst := indexNameToDestPartitionSeqs[indexName]
if dst == nil {
continue
}
var totSeq uint64
var curSeq uint64
for partitionId, dstUUIDSeq := range dst {
srcUUIDSeq, exists := src[partitionId]
if exists {
totSeq += srcUUIDSeq.Seq
curSeq += dstUUIDSeq.Seq
}
}
nsIndexStat["num_mutations_to_index"] = totSeq - curSeq
}
}
topLevelStats := map[string]interface{}{}
topLevelStats["num_bytes_used_ram"] = rd.memStats.Alloc
topLevelStats["total_gc"] = rd.memStats.NumGC
topLevelStats["pct_cpu_gc"] = rd.memStats.GCCPUFraction
nsIndexStats[""] = topLevelStats
if LogEveryNStats != 0 && currentStatsCount%int64(LogEveryNStats) == 0 {
go func() {
var buf bytes.Buffer
err := rest.WriteManagerStatsJSON(h.mgr, &buf, "")
if err != nil {
log.Printf("error formatting managerStatsJSON for logs: %v", err)
} else {
log.Printf("managerStats: %s", buf.String())
}
statsJSON, err := json.MarshalIndent(nsIndexStats, "", " ")
if err != nil {
log.Printf("error formatting JSON for logs: %v", err)
return
}
log.Printf("stats: %s", string(statsJSON))
}()
}
rest.MustEncode(w, nsIndexStats)
}
