func (ds *ElasticSearchDataSource) runQuery() {
// initial state
resultsSeen := 0
// send the query (first batch)
searchresponse, err := core.Search(true, ds.indexName, "", ds.query, "1m")
if err != nil {
log.Printf("Error running ES query: %v", err)
return
}
// extract results from the response
docs, err := ds.DocsFromSearchResults(searchresponse)
if err != nil {
log.Printf("Error parsing ES response: %v", err)
return
}
// send this batch of results
for _, doc := range docs {
ds.OutputChannel <- doc
}
// updated state after initial response
resultsSeen += len(searchresponse.Hits.Hits)
scrollId := searchresponse.ScrollId
// total results adjusted for any offset
total := searchresponse.Hits.Total - ds.offset
if ds.limit != -1 && total > ds.limit {
// if user specified a limit that is lower than the total
// we must stop at that limit
total = ds.limit
}
for resultsSeen < total {
// request the next batch
searchresponse, err := core.Scroll(true, scrollId, "1m")
if err != nil {
log.Printf("Error running ES scroll: %v", err)
return
}
// extract results from the response
docs, err := ds.DocsFromSearchResults(searchresponse)
if err != nil {
log.Printf("Error parsing ES response: %v", err)
return
}
// send this batch of results
for _, doc := range docs {
if resultsSeen < total {
ds.OutputChannel <- doc
}
resultsSeen += 1
}
scrollId = searchresponse.ScrollId
}
}
func ESDocMetaMatchingQuery(datasource string, query map[string]interface{}) ([]CouchbaseDocMeta, error) {
if *debugElasticSearch {
queryBytes, _ := json.Marshal(query)
queryString := string(queryBytes)
log.Printf("ES Query JSON is: %v", queryString)
}
result := make([]CouchbaseDocMeta, 0)
resultsSeen := 0
searchresponse, err := core.Search(true, datasource, "", query, "1m")
if err != nil {
return nil, err
}
if *debugElasticSearch {
log.Printf("Response %#v", searchresponse)
}
srmeta, err := DocMetaFromSearchResultHits(searchresponse)
if err != nil {
return nil, err
}
result = concatMetaSlices(result, srmeta)
resultsSeen += len(searchresponse.Hits.Hits)
scrollId := searchresponse.ScrollId
total := searchresponse.Hits.Total
offset_int := 0
offset, has_offset := query["from"]
if has_offset {
offset_int = offset.(int)
total = total - offset_int
}
limit, has_limit := query["size"]
if has_limit {
limit_int := limit.(int)
if limit_int < total {
total = limit_int
}
}
for resultsSeen < total {
searchresponse, err := core.Scroll(true, scrollId, "1m")
if err != nil {
return nil, err
}
if *debugElasticSearch {
log.Printf("Response %#v", searchresponse)
}
srmeta, err := DocMetaFromSearchResultHits(searchresponse)
if err != nil {
return nil, err
}
result = concatMetaSlices(result, srmeta)
resultsSeen += len(searchresponse.Hits.Hits)
scrollId = searchresponse.ScrollId
}
if len(result) != total {
return nil, errors.New("Length of result set does not match total")
}
return result, nil
}