Ejemplo n.º 1
0
func (ds *ElasticSearchDataSource) runQuery() {
	// initial state
	resultsSeen := 0

	// send the query (first batch)
	searchresponse, err := core.Search(true, ds.indexName, "", ds.query, "1m")
	if err != nil {
		log.Printf("Error running ES query: %v", err)
		return
	}

	// extract results from the response
	docs, err := ds.DocsFromSearchResults(searchresponse)
	if err != nil {
		log.Printf("Error parsing ES response: %v", err)
		return
	}

	// send this batch of results
	for _, doc := range docs {
		ds.OutputChannel <- doc
	}

	// updated state after initial response
	resultsSeen += len(searchresponse.Hits.Hits)
	scrollId := searchresponse.ScrollId
	// total results adjusted for any offset
	total := searchresponse.Hits.Total - ds.offset
	if ds.limit != -1 && total > ds.limit {
		// if user specified a limit that is lower than the total
		// we must stop at that limit
		total = ds.limit
	}

	for resultsSeen < total {
		// request the next batch
		searchresponse, err := core.Scroll(true, scrollId, "1m")
		if err != nil {
			log.Printf("Error running ES scroll: %v", err)
			return
		}

		// extract results from the response
		docs, err := ds.DocsFromSearchResults(searchresponse)
		if err != nil {
			log.Printf("Error parsing ES response: %v", err)
			return
		}

		// send this batch of results
		for _, doc := range docs {
			if resultsSeen < total {
				ds.OutputChannel <- doc
			}
			resultsSeen += 1
		}

		scrollId = searchresponse.ScrollId
	}
}
Ejemplo n.º 2
0
func ESDocMetaMatchingQuery(datasource string, query map[string]interface{}) ([]CouchbaseDocMeta, error) {

	if *debugElasticSearch {
		queryBytes, _ := json.Marshal(query)
		queryString := string(queryBytes)
		log.Printf("ES Query JSON is: %v", queryString)
	}

	result := make([]CouchbaseDocMeta, 0)

	resultsSeen := 0
	searchresponse, err := core.Search(true, datasource, "", query, "1m")
	if err != nil {
		return nil, err
	}

	if *debugElasticSearch {
		log.Printf("Response %#v", searchresponse)
	}

	srmeta, err := DocMetaFromSearchResultHits(searchresponse)
	if err != nil {
		return nil, err
	}
	result = concatMetaSlices(result, srmeta)

	resultsSeen += len(searchresponse.Hits.Hits)
	scrollId := searchresponse.ScrollId
	total := searchresponse.Hits.Total

	offset_int := 0
	offset, has_offset := query["from"]
	if has_offset {
		offset_int = offset.(int)
		total = total - offset_int
	}

	limit, has_limit := query["size"]
	if has_limit {
		limit_int := limit.(int)
		if limit_int < total {
			total = limit_int
		}
	}

	for resultsSeen < total {
		searchresponse, err := core.Scroll(true, scrollId, "1m")
		if err != nil {
			return nil, err
		}

		if *debugElasticSearch {
			log.Printf("Response %#v", searchresponse)
		}

		srmeta, err := DocMetaFromSearchResultHits(searchresponse)
		if err != nil {
			return nil, err
		}
		result = concatMetaSlices(result, srmeta)

		resultsSeen += len(searchresponse.Hits.Hits)
		scrollId = searchresponse.ScrollId

	}

	if len(result) != total {
		return nil, errors.New("Length of result set does not match total")
	}

	return result, nil
}