Example #1
0
func QueryBlevePIndexImpl(mgr *cbgt.Manager, indexName, indexUUID string,
	req []byte, res io.Writer) error {
	queryCtlParams := cbgt.QueryCtlParams{
		Ctl: cbgt.QueryCtl{
			Timeout: cbgt.QUERY_CTL_DEFAULT_TIMEOUT_MS,
		},
	}

	err := json.Unmarshal(req, &queryCtlParams)
	if err != nil {
		return fmt.Errorf("bleve: QueryBlevePIndexImpl"+
			" parsing queryCtlParams, req: %s, err: %v", req, err)
	}

	searchRequest := &bleve.SearchRequest{}

	err = json.Unmarshal(req, searchRequest)
	if err != nil {
		return fmt.Errorf("bleve: QueryBlevePIndexImpl"+
			" parsing searchRequest, req: %s, err: %v", req, err)
	}

	err = searchRequest.Query.Validate()
	if err != nil {
		return err
	}

	cancelCh := cbgt.TimeoutCancelChan(queryCtlParams.Ctl.Timeout)

	alias, err := bleveIndexAlias(mgr, indexName, indexUUID, true,
		queryCtlParams.Ctl.Consistency, cancelCh)
	if err != nil {
		return err
	}

	doneCh := make(chan struct{})

	var searchResult *bleve.SearchResult

	go func() {
		searchResult, err = alias.Search(searchRequest)

		close(doneCh)
	}()

	select {
	case <-cancelCh:
		err = fmt.Errorf("pindex_bleve: query timeout")

	case <-doneCh:
		if searchResult != nil {
			rest.MustEncode(res, searchResult)
		}
	}

	return err
}
Example #2
0
func (h *NsStatusHandler) ServeHTTP(
	w http.ResponseWriter, req *http.Request) {
	initNsServerCaching(h.mgr)

	rd := <-recentInfoCh
	if rd.err != nil {
		rest.ShowError(w, req, fmt.Sprintf("could not retrieve defs: %v", rd.err), 500)
		return
	}

	indexDefsMap := rd.indexDefsMap
	nodeDefs := rd.nodeDefs
	planPIndexes := rd.planPIndexes

	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Write(cbgt.JsonOpenBrace)
	w.Write(statsNamePrefix)
	w.Write([]byte("status"))
	w.Write(statsNameSuffix)
	w.Write([]byte("["))

	indexDefNames := make(sort.StringSlice, 0, len(indexDefsMap))
	for indexDefName := range indexDefsMap {
		indexDefNames = append(indexDefNames, indexDefName)
	}

	sort.Sort(indexDefNames)

	for i, indexDefName := range indexDefNames {
		indexDef := indexDefsMap[indexDefName]
		if i > 0 {
			w.Write(cbgt.JsonComma)
		}

		rest.MustEncode(w, struct {
			Hosts  []string `json:"hosts"`
			Bucket string   `json:"bucket"`
			Name   string   `json:"name"`
		}{
			Bucket: indexDef.SourceName,
			Name:   indexDefName,
			Hosts:  NsHostsForIndex(indexDefName, planPIndexes, nodeDefs),
		})
	}

	w.Write([]byte("],"))
	w.Write(statsNamePrefix)
	w.Write([]byte("code"))
	w.Write(statsNameSuffix)
	w.Write([]byte("\"success\""))
	w.Write(cbgt.JsonCloseBrace)
}
func (t *BleveDest) Query(pindex *cbgt.PIndex, req []byte, res io.Writer,
	cancelCh <-chan bool) error {
	queryCtlParams := cbgt.QueryCtlParams{
		Ctl: cbgt.QueryCtl{
			Timeout: cbgt.QUERY_CTL_DEFAULT_TIMEOUT_MS,
		},
	}

	err := json.Unmarshal(req, &queryCtlParams)
	if err != nil {
		return fmt.Errorf("bleve: BleveDest.Query"+
			" parsing queryCtlParams, req: %s, err: %v", req, err)
	}

	searchRequest := &bleve.SearchRequest{}

	err = json.Unmarshal(req, searchRequest)
	if err != nil {
		return fmt.Errorf("bleve: BleveDest.Query"+
			" parsing searchRequest, req: %s, err: %v", req, err)
	}

	err = cbgt.ConsistencyWaitPIndex(pindex, t,
		queryCtlParams.Ctl.Consistency, cancelCh)
	if err != nil {
		return err
	}

	err = searchRequest.Query.Validate()
	if err != nil {
		return err
	}

	t.m.Lock()
	bindex := t.bindex
	t.m.Unlock()

	if bindex == nil {
		return fmt.Errorf("bleve: Query, bindex already closed")
	}

	searchResponse, err := bindex.Search(searchRequest)
	if err != nil {
		return err
	}

	rest.MustEncode(res, searchResponse)

	return nil
}
Example #4
0
func QueryAlias(mgr *cbgt.Manager, indexName, indexUUID string,
	req []byte, res io.Writer) error {
	queryCtlParams := cbgt.QueryCtlParams{
		Ctl: cbgt.QueryCtl{
			Timeout: cbgt.QUERY_CTL_DEFAULT_TIMEOUT_MS,
		},
	}

	err := json.Unmarshal(req, &queryCtlParams)
	if err != nil {
		return fmt.Errorf("alias: QueryAlias"+
			" parsing queryCtlParams, req: %s, err: %v", req, err)
	}

	searchRequest := &bleve.SearchRequest{}

	err = json.Unmarshal(req, searchRequest)
	if err != nil {
		return fmt.Errorf("alias: QueryAlias"+
			" parsing searchRequest, req: %s, err: %v", req, err)
	}

	if srqv, ok := searchRequest.Query.(query.ValidatableQuery); ok {
		err = srqv.Validate()
		if err != nil {
			return err
		}
	}

	cancelCh := cbgt.TimeoutCancelChan(queryCtlParams.Ctl.Timeout)

	alias, err := bleveIndexAliasForUserIndexAlias(mgr,
		indexName, indexUUID, true,
		queryCtlParams.Ctl.Consistency, cancelCh, true)
	if err != nil {
		return err
	}

	searchResponse, err := alias.Search(searchRequest)
	if err != nil {
		return err
	}

	rest.MustEncode(res, searchResponse)

	return nil
}
Example #5
0
func CheckAPIVersion(w http.ResponseWriter, req *http.Request) (err error) {
	var version = API_MAX_VERSION

	if req.Header != nil && len(req.Header["Accept"]) > 0 {
		version, err = HandleAPIVersion(req.Header["Accept"][0])
		if err != nil {
			w.WriteHeader(406)

			versionList := []string{
				WithJSONVersion(API_MAX_VERSION),
				WithJSONVersion(API_MIN_VERSION),
			}

			rest.MustEncode(w, versionList)

			return err
		}
	}

	w.Header().Set("Content-type", WithJSONVersion(version))

	return nil
}
Example #6
0
func (h *NsStatsHandler) ServeHTTP(
	w http.ResponseWriter, req *http.Request) {

	_, indexDefsMap, err := h.mgr.GetIndexDefs(false)
	if err != nil {
		rest.ShowError(w, req, "could not retrieve index defs", 500)
		return
	}

	nsIndexStats := make(NSIndexStats, len(indexDefsMap))
	for indexDefName, indexDef := range indexDefsMap {
		nsIndexStats[indexDef.SourceName+":"+indexDefName] = NewIndexStat()
	}

	feeds, pindexes := h.mgr.CurrentMaps()

	sourceName := ""
	for _, pindex := range pindexes {
		sourceName = pindex.SourceName
		lindexName := pindex.SourceName + ":" + pindex.IndexName
		nsIndexStat, ok := nsIndexStats[lindexName]
		if ok {

			// manually track a statistic representing
			// the number of pindex in the index
			oldValue, ok := nsIndexStat["num_pindexes"]
			if ok {
				switch oldValue := oldValue.(type) {
				case float64:
					oldValue += float64(1)
					nsIndexStat["num_pindexes"] = oldValue
				}
			}

			// automatically process all the pindex dest stats
			err := addPindexStats(pindex, nsIndexStat)
			if err != nil {
				rest.ShowError(w, req, fmt.Sprintf("error processing PIndex stats: %v", err), 500)
				return
			}
		}
	}

	for _, feed := range feeds {
		lindexName := sourceName + ":" + feed.IndexName()
		nsIndexStat, ok := nsIndexStats[lindexName]
		if ok {
			err := addFeedStats(feed, nsIndexStat)

			// automatically process all the feed stats
			if err != nil {
				rest.ShowError(w, req, fmt.Sprintf("error processing Feed stats: %v", err), 500)
				return
			}
		}
	}

	// FIXME hard-coded top-level stats
	nsIndexStats[""] = make(map[string]interface{})
	nsIndexStats[""]["num_connections"] = 0
	nsIndexStats[""]["needs_restart"] = false

	rest.MustEncode(w, nsIndexStats)
}
Example #7
0
func (h *NsStatusHandler) ServeHTTP(
	w http.ResponseWriter, req *http.Request) {

	cfg := h.mgr.Cfg()
	planPIndexes, _, err := cbgt.CfgGetPlanPIndexes(cfg)
	if err != nil {
		rest.ShowError(w, req, "could not retrieve plan pIndexes", 500)
		return
	}

	nodesDefs, _, err := cbgt.CfgGetNodeDefs(cfg, cbgt.NODE_DEFS_WANTED)
	if err != nil {
		rest.ShowError(w, req, "could not retrieve node defs (wanted)", 500)
		return
	}

	_, indexDefsMap, err := h.mgr.GetIndexDefs(false)
	if err != nil {
		rest.ShowError(w, req, "could not retrieve index defs", 500)
		return
	}

	w.Write(cbgt.JsonOpenBrace)
	w.Write(statsNamePrefix)
	w.Write([]byte("status"))
	w.Write(statsNameSuffix)
	w.Write([]byte("["))

	indexDefNames := make(sort.StringSlice, 0, len(indexDefsMap))
	for indexDefName := range indexDefsMap {
		indexDefNames = append(indexDefNames, indexDefName)
	}

	sort.Sort(indexDefNames)

	for i, indexDefName := range indexDefNames {
		indexDef := indexDefsMap[indexDefName]
		if i > 0 {
			w.Write(cbgt.JsonComma)
		}

		rest.MustEncode(w, struct {
			Completion int      `json:"completion"`
			Hosts      []string `json:"hosts"`
			Status     string   `json:"status"`
			Bucket     string   `json:"bucket"`
			Name       string   `json:"name"`
		}{
			Bucket: indexDef.SourceName,
			Name:   indexDefName,
			Hosts:  HostsForIndex(indexDefName, planPIndexes, nodesDefs),
			// FIXME hard-coded
			Completion: 100,
			Status:     "Ready",
		})

	}
	w.Write([]byte("],"))
	w.Write(statsNamePrefix)
	w.Write([]byte("code"))
	w.Write(statsNameSuffix)
	w.Write([]byte("\"success\""))
	w.Write(cbgt.JsonCloseBrace)

}
Example #8
0
func (h *NsStatsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	currentStatsCount := atomic.AddInt64(&h.statsCount, 1)
	initNsServerCaching(h.mgr)

	rd := <-recentInfoCh
	if rd.err != nil {
		rest.ShowError(w, req, fmt.Sprintf("could not retrieve defs: %v", rd.err), 500)
		return
	}

	indexDefsMap := rd.indexDefsMap
	planPIndexes := rd.planPIndexes

	nodeUUID := h.mgr.UUID()

	nsIndexStats := make(NSIndexStats, len(indexDefsMap))

	indexNameToSourceName := map[string]string{}

	indexNameToPlanPIndexes := map[string][]*cbgt.PlanPIndex{}
	if planPIndexes != nil {
		for _, planPIndex := range planPIndexes.PlanPIndexes {
			// Only focus on the planPIndex entries for this node.
			if planPIndex.Nodes[nodeUUID] != nil {
				indexNameToPlanPIndexes[planPIndex.IndexName] =
					append(indexNameToPlanPIndexes[planPIndex.IndexName], planPIndex)
			}
		}
	}

	indexQueryPathStats := MapRESTPathStats[RESTIndexQueryPath]

	// Keyed by indexName, sub-key is source partition id.
	indexNameToSourcePartitionSeqs := map[string]map[string]cbgt.UUIDSeq{}

	// Keyed by indexName, sub-key is source partition id.
	indexNameToDestPartitionSeqs := map[string]map[string]cbgt.UUIDSeq{}

	for indexName, indexDef := range indexDefsMap {
		nsIndexStat := NewIndexStat()
		nsIndexStats[indexDef.SourceName+":"+indexName] = nsIndexStat

		indexNameToSourceName[indexName] = indexDef.SourceName

		focusStats := indexQueryPathStats.FocusStats(indexName)
		if focusStats != nil {
			totalQueries := atomic.LoadUint64(&focusStats.TotRequest)
			nsIndexStat["total_queries"] = totalQueries
			if totalQueries > 0 {
				nsIndexStat["avg_queries_latency"] =
					float64((atomic.LoadUint64(&focusStats.TotRequestTimeNS) /
						totalQueries)) / 1000000.0 // Convert from nanosecs to millisecs.
			}
			nsIndexStat["total_request_time"] =
				atomic.LoadUint64(&focusStats.TotRequestTimeNS)
			nsIndexStat["total_queries_slow"] =
				atomic.LoadUint64(&focusStats.TotRequestSlow)
			nsIndexStat["total_queries_timeout"] =
				atomic.LoadUint64(&focusStats.TotRequestTimeout)
			nsIndexStat["total_queries_error"] =
				atomic.LoadUint64(&focusStats.TotRequestErr)
			nsIndexStat["total_bytes_query_results"] =
				atomic.LoadUint64(&focusStats.TotResponseBytes)
			nsIndexStat["num_pindexes_target"] =
				uint64(len(indexNameToPlanPIndexes[indexName]))
		}

		feedType, exists := cbgt.FeedTypes[indexDef.SourceType]
		if !exists || feedType == nil || feedType.PartitionSeqs == nil {
			continue
		}

		partitionSeqs := GetSourcePartitionSeqs(SourceSpec{
			SourceType:   indexDef.SourceType,
			SourceName:   indexDef.SourceName,
			SourceUUID:   indexDef.SourceUUID,
			SourceParams: indexDef.SourceParams,
			Server:       h.mgr.Server(),
		})
		if partitionSeqs != nil {
			indexNameToSourcePartitionSeqs[indexName] = partitionSeqs
		}
	}

	feeds, pindexes := h.mgr.CurrentMaps()

	for _, pindex := range pindexes {
		nsIndexName := pindex.SourceName + ":" + pindex.IndexName
		nsIndexStat, ok := nsIndexStats[nsIndexName]
		if ok {
			// manually track num pindexes
			oldValue, ok := nsIndexStat["num_pindexes_actual"]
			if ok {
				switch oldValue := oldValue.(type) {
				case float64:
					oldValue += float64(1)

					nsIndexStat["num_pindexes_actual"] = oldValue

					// TODO: Former name was num_pindexes, need to remove one day.
					nsIndexStat["num_pindexes"] = oldValue
				}
			}

			// automatically process all the pindex dest stats
			err := addPIndexStats(pindex, nsIndexStat)
			if err != nil {
				rest.ShowError(w, req,
					fmt.Sprintf("error processing PIndex stats: %v", err), 500)
				return
			}

			dest := pindex.Dest
			if dest != nil {
				destForwarder, ok := dest.(*cbgt.DestForwarder)
				if !ok {
					continue
				}

				partitionSeqsProvider, ok :=
					destForwarder.DestProvider.(PartitionSeqsProvider)
				if !ok {
					continue
				}

				partitionSeqs, err := partitionSeqsProvider.PartitionSeqs()
				if err == nil {
					m := indexNameToDestPartitionSeqs[pindex.IndexName]
					if m == nil {
						m = map[string]cbgt.UUIDSeq{}
						indexNameToDestPartitionSeqs[pindex.IndexName] = m
					}

					for partitionId, uuidSeq := range partitionSeqs {
						m[partitionId] = uuidSeq
					}
				}
			}
		}
	}

	for _, feed := range feeds {
		sourceName := indexNameToSourceName[feed.IndexName()]
		nsIndexName := sourceName + ":" + feed.IndexName()
		nsIndexStat, ok := nsIndexStats[nsIndexName]
		if ok {
			// automatically process all the feed stats
			err := addFeedStats(feed, nsIndexStat)
			if err != nil {
				rest.ShowError(w, req,
					fmt.Sprintf("error processing Feed stats: %v", err), 500)
				return
			}
		}
	}

	for indexName, indexDef := range indexDefsMap {
		nsIndexStat, ok := nsIndexStats[indexDef.SourceName+":"+indexName]
		if ok {
			src := indexNameToSourcePartitionSeqs[indexName]
			if src == nil {
				continue
			}

			dst := indexNameToDestPartitionSeqs[indexName]
			if dst == nil {
				continue
			}

			var totSeq uint64
			var curSeq uint64

			for partitionId, dstUUIDSeq := range dst {
				srcUUIDSeq, exists := src[partitionId]
				if exists {
					totSeq += srcUUIDSeq.Seq
					curSeq += dstUUIDSeq.Seq
				}
			}

			nsIndexStat["num_mutations_to_index"] = totSeq - curSeq
		}
	}

	topLevelStats := map[string]interface{}{}

	topLevelStats["num_bytes_used_ram"] = rd.memStats.Alloc
	topLevelStats["total_gc"] = rd.memStats.NumGC
	topLevelStats["pct_cpu_gc"] = rd.memStats.GCCPUFraction

	nsIndexStats[""] = topLevelStats

	if LogEveryNStats != 0 && currentStatsCount%int64(LogEveryNStats) == 0 {
		go func() {
			var buf bytes.Buffer
			err := rest.WriteManagerStatsJSON(h.mgr, &buf, "")
			if err != nil {
				log.Printf("error formatting managerStatsJSON for logs: %v", err)
			} else {
				log.Printf("managerStats: %s", buf.String())
			}

			statsJSON, err := json.MarshalIndent(nsIndexStats, "", "    ")
			if err != nil {
				log.Printf("error formatting JSON for logs: %v", err)
				return
			}
			log.Printf("stats: %s", string(statsJSON))
		}()
	}

	rest.MustEncode(w, nsIndexStats)
}