func ExampleClient_NewClient_cluster() {
// Obtain a client for an Elasticsearch cluster of two nodes,
// running on 10.0.1.1 and 10.0.1.2.
client, err := elastic.NewClient(elastic.SetURL("http://10.0.1.1:9200", "http://10.0.1.2:9200"))
if err != nil {
// Handle error
panic(err)
}
_ = client
}
// InitClient sets up the elastic client. If the client has already been
// initalized it is a noop
func (e ElasticHosts) InitClient() error {
if esClient == nil {
var err error
esClient, err = elastic.NewClient(elastic.SetURL(e...), elastic.SetMaxRetries(10))
if err != nil {
return err
}
}
return nil
}
func (e *Elastic) InitBackend() error {
var err error
var ec *elastic.Client
if e.simpleClient {
ec, err = elastic.NewSimpleClient(elastic.SetURL(e.urls...))
} else {
ec, err = elastic.NewClient(elastic.SetURL(e.urls...))
}
if err != nil {
return err
}
e.Client = ec
exists, err := e.IndexExists(e.index).Do()
if err != nil {
return err
}
if !exists {
res, err := e.CreateIndex(e.index).Do()
if (res != nil && !res.Acknowledged) || err != nil {
return fmt.Errorf("failed to create elastic mapping (ack: %v): %v", res != nil && res.Acknowledged, err)
}
}
stringNA := map[string]string{
"type": "string",
"index": "not_analyzed",
}
stringA := map[string]string{
"type": "string",
}
date := map[string]string{
"type": "date",
}
p := make(map[string]interface{})
p[annotate.Message] = stringA
p[annotate.StartDate] = date
p[annotate.EndDate] = date
p[annotate.Source] = stringNA
p[annotate.Host] = stringNA
p[annotate.CreationUser] = stringNA
p[annotate.Owner] = stringNA
p[annotate.Category] = stringNA
p[annotate.Url] = stringNA
mapping := make(map[string]interface{})
mapping["properties"] = p
q := e.PutMapping().Index(e.index).Type(docType).BodyJson(mapping)
res, err := q.Do()
if (res != nil && !res.Acknowledged) || err != nil {
return fmt.Errorf("failed to create elastic mapping (ack: %v): %v", res != nil && res.Acknowledged, err)
}
e.initialized = true
return nil
}
func New() (*ElasticWReport, error) {
client, err := el.NewClient()
if err != nil {
return nil, fmt.Errorf("NewClient: %s", err.Error())
}
info, code, err := client.Ping(el.DefaultURL).Do()
if err != nil {
return nil, fmt.Errorf("Ping: %s", err.Error())
}
fmt.Printf("Elasticsearch returned with code %d and version %s", code, info.Version.Number)
return &ElasticWReport{client, "", ""}, nil
}
func ExampleClient_NewClient_default() {
// Obtain a client to the Elasticsearch instance on http://127.0.0.1:9200.
client, err := elastic.NewClient()
if err != nil {
// Handle error
fmt.Printf("connection failed: %v\n", err)
} else {
fmt.Println("connected")
}
_ = client
// Output:
// connected
}
func ExampleGetTemplateService() {
client, err := elastic.NewClient()
if err != nil {
panic(err)
}
// Get template stored under "my-search-template"
resp, err := client.GetTemplate().Id("my-search-template").Do()
if err != nil {
panic(err)
}
fmt.Printf("search template is: %q\n", resp.Template)
}
func ExampleClusterStateService() {
client, err := elastic.NewClient()
if err != nil {
panic(err)
}
// Get cluster state
res, err := client.ClusterState().Metric("version").Do()
if err != nil {
panic(err)
}
fmt.Printf("Cluster %q has version %d", res.ClusterName, res.Version)
}
func ExampleSearchResult() {
client, err := elastic.NewClient()
if err != nil {
panic(err)
}
// Do a search
searchResult, err := client.Search().Index("twitter").Query(elastic.NewMatchAllQuery()).Do()
if err != nil {
panic(err)
}
// searchResult is of type SearchResult and returns hits, suggestions,
// and all kinds of other information from Elasticsearch.
fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis)
// Each is a utility function that iterates over hits in a search result.
// It makes sure you don't need to check for nil values in the response.
// However, it ignores errors in serialization. If you want full control
// over iterating the hits, see below.
var ttyp Tweet
for _, item := range searchResult.Each(reflect.TypeOf(ttyp)) {
t := item.(Tweet)
fmt.Printf("Tweet by %s: %s\n", t.User, t.Message)
}
fmt.Printf("Found a total of %d tweets\n", searchResult.TotalHits())
// Here's how you iterate hits with full control.
if searchResult.Hits.TotalHits > 0 {
fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits)
// Iterate through results
for _, hit := range searchResult.Hits.Hits {
// hit.Index contains the name of the index
// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
var t Tweet
err := json.Unmarshal(*hit.Source, &t)
if err != nil {
// Deserialization failed
}
// Work with tweet
fmt.Printf("Tweet by %s: %s\n", t.User, t.Message)
}
} else {
// No hits
fmt.Print("Found no tweets\n")
}
}
func ExampleSearchService() {
// Get a client to the local Elasticsearch instance.
client, err := elastic.NewClient()
if err != nil {
// Handle error
panic(err)
}
// Search with a term query
termQuery := elastic.NewTermQuery("user", "olivere")
searchResult, err := client.Search().
Index("twitter"). // search in index "twitter"
Query(termQuery). // specify the query
Sort("user", true). // sort by "user" field, ascending
From(0).Size(10). // take documents 0-9
Pretty(true). // pretty print request and response JSON
Do() // execute
if err != nil {
// Handle error
panic(err)
}
// searchResult is of type SearchResult and returns hits, suggestions,
// and all kinds of other information from Elasticsearch.
fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis)
// Number of hits
if searchResult.Hits.TotalHits > 0 {
fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits)
// Iterate through results
for _, hit := range searchResult.Hits.Hits {
// hit.Index contains the name of the index
// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
var t Tweet
err := json.Unmarshal(*hit.Source, &t)
if err != nil {
// Deserialization failed
}
// Work with tweet
fmt.Printf("Tweet by %s: %s\n", t.User, t.Message)
}
} else {
// No hits
fmt.Print("Found no tweets\n")
}
}
func ExampleDeleteTemplateService() {
client, err := elastic.NewClient()
if err != nil {
panic(err)
}
// Delete template
resp, err := client.DeleteTemplate().Id("my-search-template").Do()
if err != nil {
panic(err)
}
if resp != nil && resp.Found {
fmt.Println("template deleted")
}
}
func ExampleClusterHealthService() {
client, err := elastic.NewClient()
if err != nil {
panic(err)
}
// Get cluster health
res, err := client.ClusterHealth().Index("twitter").Do()
if err != nil {
panic(err)
}
if res == nil {
panic(err)
}
fmt.Printf("Cluster status is %q\n", res.Status)
}
func ExampleIndexExistsService() {
// Get a client to the local Elasticsearch instance.
client, err := elastic.NewClient()
if err != nil {
// Handle error
panic(err)
}
// Use the IndexExists service to check if the index "twitter" exists.
exists, err := client.IndexExists("twitter").Do()
if err != nil {
// Handle error
panic(err)
}
if exists {
// ...
}
}
func ExampleClusterHealthService_WaitForGreen() {
client, err := elastic.NewClient()
if err != nil {
panic(err)
}
// Wait for status green
res, err := client.ClusterHealth().WaitForStatus("green").Timeout("15s").Do()
if err != nil {
panic(err)
}
if res.TimedOut {
fmt.Printf("time out waiting for cluster status %q\n", "green")
} else {
fmt.Printf("cluster status is %q\n", res.Status)
}
}
func ExampleDeleteIndexService() {
// Get a client to the local Elasticsearch instance.
client, err := elastic.NewClient()
if err != nil {
// Handle error
panic(err)
}
// Delete an index.
deleteIndex, err := client.DeleteIndex("twitter").Do()
if err != nil {
// Handle error
panic(err)
}
if !deleteIndex.Acknowledged {
// Not acknowledged
}
}
func ExampleAggregations() {
// Get a client to the local Elasticsearch instance.
client, err := elastic.NewClient()
if err != nil {
// Handle error
panic(err)
}
// Create an aggregation for users and a sub-aggregation for a date histogram of tweets (per year).
timeline := elastic.NewTermsAggregation().Field("user").Size(10).OrderByCountDesc()
histogram := elastic.NewDateHistogramAggregation().Field("created").Interval("year")
timeline = timeline.SubAggregation("history", histogram)
// Search with a term query
searchResult, err := client.Search().
Index("twitter"). // search in index "twitter"
Query(elastic.NewMatchAllQuery()). // return all results, but ...
SearchType("count"). // ... do not return hits, just the count
Aggregation("timeline", timeline). // add our aggregation to the query
Pretty(true). // pretty print request and response JSON
Do() // execute
if err != nil {
// Handle error
panic(err)
}
// Access "timeline" aggregate in search result.
agg, found := searchResult.Aggregations.Terms("timeline")
if !found {
log.Fatalf("we sould have a terms aggregation called %q", "timeline")
}
for _, userBucket := range agg.Buckets {
// Every bucket should have the user field as key.
user := userBucket.Key
// The sub-aggregation history should have the number of tweets per year.
histogram, found := userBucket.DateHistogram("history")
if found {
for _, year := range histogram.Buckets {
fmt.Printf("user %q has %d tweets in %q\n", user, year.DocCount, year.KeyAsString)
}
}
}
}
func ExampleClient_NewClient_manyOptions() {
// Obtain a client for an Elasticsearch cluster of two nodes,
// running on 10.0.1.1 and 10.0.1.2. Do not run the sniffer.
// Set the healthcheck interval to 10s. When requests fail,
// retry 5 times. Print error messages to os.Stderr and informational
// messages to os.Stdout.
client, err := elastic.NewClient(
elastic.SetURL("http://10.0.1.1:9200", "http://10.0.1.2:9200"),
elastic.SetSniff(false),
elastic.SetHealthcheckInterval(10*time.Second),
elastic.SetMaxRetries(5),
elastic.SetErrorLog(log.New(os.Stderr, "ELASTIC ", log.LstdFlags)),
elastic.SetInfoLog(log.New(os.Stdout, "", log.LstdFlags)))
if err != nil {
// Handle error
panic(err)
}
_ = client
}
func newIndexer(config Configuration) *Indexer {
f, err := os.Open(config.input)
if err != nil {
log.Critical("Unable to read input file: %v", err)
os.Exit(1)
}
e, err := elastic.NewClient(
elastic.SetURL(config.url))
if err != nil {
log.Critical("Unable to connect to the Elasticsearch servers at %s: %v", config.url, err)
os.Exit(1)
}
return &Indexer{
index: config.index,
file: f,
scaner: bufio.NewScanner(f),
es: e,
}
}
func esClient() (*es.Client, error) {
urls := strings.Split(*esURL, ",")
options := []es.ClientOptionFunc{
es.SetURL(urls...),
es.SetSniff(false),
}
if *esUser != "" {
options = append(options, es.SetBasicAuth(*esUser, *esPass))
}
if *esDebug {
l := logger{}
options = append(options,
es.SetErrorLog(l),
es.SetInfoLog(l),
es.SetTraceLog(l),
)
}
client, err := es.NewClient(options...)
if err != nil {
return nil, err
}
exists, err := client.IndexExists(*esIndex).Do()
if err != nil {
return nil, err
}
if !exists {
resp, err := client.CreateIndex(*esIndex).Body(mapping).Do()
if err != nil {
return nil, err
}
if !resp.Acknowledged {
return nil, errors.New("index creation was not acknowledged")
}
logrus.Info("index created")
}
return client, nil
}
func ExamplePutTemplateService() {
client, err := elastic.NewClient()
if err != nil {
panic(err)
}
// Create search template
tmpl := `{"template":{"query":{"match":{"title":"{{query_string}}"}}}}`
// Create template
resp, err := client.PutTemplate().
Id("my-search-template"). // Name of the template
BodyString(tmpl). // Search template itself
Do() // Execute
if err != nil {
panic(err)
}
if resp.Created {
fmt.Println("search template created")
}
}
func initElastic(www string) (*elastic.Client, error) {
username, password, host := parseBonsaiURL(www)
log.Infof("Initializing ES: %v.", host)
client, err := elastic.NewClient(elastic.SetURL(host), elastic.SetMaxRetries(10), elastic.SetBasicAuth(username, password), elastic.SetSniff(false))
if err != nil {
log.Fatalf("Error while connecting to ElasticSearch: %s", err)
return nil, err
}
log.Info("Initializing Indices.")
err = initIndices(client)
if err != nil {
log.Fatalf("Error while creating ElasticSearch Indices: %s", err)
return nil, err
}
return client, err
}
func NewElastic(urls []string, index string) (*Elastic, error) {
e, err := elastic.NewClient(elastic.SetURL(urls...))
return &Elastic{e, index, 200}, err
}
// Returns object to work with bleve
func NewElasticStorage() *Elastic {
url, err := config.Instance().String("elastic", "url")
if err != nil {
logger.Instance().
WithError(err).
Error("Elastic url is not provided")
os.Exit(1)
}
client, err := es.NewClient(
es.SetURL(url),
es.SetSniff(false),
es.SetHealthcheck(false),
es.SetMaxRetries(0),
)
if err != nil {
logger.Instance().
WithError(err).
Error("Unable to create client to elastic")
os.Exit(1)
}
indexName, err := config.Instance().String("elastic", "index")
if err != nil {
logger.Instance().
WithError(err).
Error("Index name is not provided")
os.Exit(1)
}
typeName, err := config.Instance().String("elastic", "type")
if err != nil {
logger.Instance().
WithError(err).
Error("Type name is not provided")
os.Exit(1)
}
batchSize, err := config.Instance().Int("elastic", "batch_size")
if err != nil {
batchSize = 10
}
var (
defaultIntervalSecond string = "1s"
defaultIntervalMonth string = "720h"
)
intervalCleanupStr, err := config.Instance().String("elastic", "interval_cleanup")
if err != nil {
intervalCleanupStr = defaultIntervalMonth
}
intervalCleanup, err := time.ParseDuration(intervalCleanupStr)
if err != nil {
intervalCleanup, _ = time.ParseDuration(defaultIntervalMonth)
}
intervalFlushStr, err := config.Instance().String("elastic", "interval_flush")
if err != nil {
intervalFlushStr = defaultIntervalSecond
}
intervalFlush, err := time.ParseDuration(intervalFlushStr)
if err != nil {
intervalFlush, _ = time.ParseDuration(defaultIntervalSecond)
}
return &Elastic{
batchSize: batchSize,
indexName: indexName,
typeName: typeName,
client: client,
messages: []*storage.Message{},
mutexHandleMessage: &sync.RWMutex{},
mutexFlushMessages: &sync.RWMutex{},
ttl: int64(intervalCleanup.Seconds() * 1000), // TTL is in milliseconds
intervalFlush: intervalFlush,
lastFlush: time.Now(),
}
}
func Example() {
errorlog := log.New(os.Stdout, "APP ", log.LstdFlags)
// Obtain a client. You can also provide your own HTTP client here.
client, err := elastic.NewClient(elastic.SetErrorLog(errorlog))
if err != nil {
// Handle error
panic(err)
}
// Trace request and response details like this
//client.SetTracer(log.New(os.Stdout, "", 0))
// Ping the Elasticsearch server to get e.g. the version number
info, code, err := client.Ping("http://127.0.0.1:9200").Do()
if err != nil {
// Handle error
panic(err)
}
fmt.Printf("Elasticsearch returned with code %d and version %s", code, info.Version.Number)
// Getting the ES version number is quite common, so there's a shortcut
esversion, err := client.ElasticsearchVersion("http://127.0.0.1:9200")
if err != nil {
// Handle error
panic(err)
}
fmt.Printf("Elasticsearch version %s", esversion)
// Use the IndexExists service to check if a specified index exists.
exists, err := client.IndexExists("twitter").Do()
if err != nil {
// Handle error
panic(err)
}
if !exists {
// Create a new index.
createIndex, err := client.CreateIndex("twitter").Do()
if err != nil {
// Handle error
panic(err)
}
if !createIndex.Acknowledged {
// Not acknowledged
}
}
// Index a tweet (using JSON serialization)
tweet1 := Tweet{User: "******", Message: "Take Five", Retweets: 0}
put1, err := client.Index().
Index("twitter").
Type("tweet").
Id("1").
BodyJson(tweet1).
Do()
if err != nil {
// Handle error
panic(err)
}
fmt.Printf("Indexed tweet %s to index %s, type %s\n", put1.Id, put1.Index, put1.Type)
// Index a second tweet (by string)
tweet2 := `{"user" : "olivere", "message" : "It's a Raggy Waltz"}`
put2, err := client.Index().
Index("twitter").
Type("tweet").
Id("2").
BodyString(tweet2).
Do()
if err != nil {
// Handle error
panic(err)
}
fmt.Printf("Indexed tweet %s to index %s, type %s\n", put2.Id, put2.Index, put2.Type)
// Get tweet with specified ID
get1, err := client.Get().
Index("twitter").
Type("tweet").
Id("1").
Do()
if err != nil {
// Handle error
panic(err)
}
if get1.Found {
fmt.Printf("Got document %s in version %d from index %s, type %s\n", get1.Id, get1.Version, get1.Index, get1.Type)
}
// Flush to make sure the documents got written.
_, err = client.Flush().Index("twitter").Do()
if err != nil {
panic(err)
}
// Search with a term query
termQuery := elastic.NewTermQuery("user", "olivere")
searchResult, err := client.Search().
Index("twitter"). // search in index "twitter"
Query(termQuery). // specify the query
Sort("user", true). // sort by "user" field, ascending
From(0).Size(10). // take documents 0-9
Pretty(true). // pretty print request and response JSON
Do() // execute
if err != nil {
// Handle error
panic(err)
}
// searchResult is of type SearchResult and returns hits, suggestions,
// and all kinds of other information from Elasticsearch.
fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis)
// Each is a convenience function that iterates over hits in a search result.
// It makes sure you don't need to check for nil values in the response.
// However, it ignores errors in serialization. If you want full control
// over iterating the hits, see below.
var ttyp Tweet
for _, item := range searchResult.Each(reflect.TypeOf(ttyp)) {
t := item.(Tweet)
fmt.Printf("Tweet by %s: %s\n", t.User, t.Message)
}
// TotalHits is another convenience function that works even when something goes wrong.
fmt.Printf("Found a total of %d tweets\n", searchResult.TotalHits())
// Here's how you iterate through results with full control over each step.
if searchResult.Hits.TotalHits > 0 {
fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits)
// Iterate through results
for _, hit := range searchResult.Hits.Hits {
// hit.Index contains the name of the index
// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
var t Tweet
err := json.Unmarshal(*hit.Source, &t)
if err != nil {
// Deserialization failed
}
// Work with tweet
fmt.Printf("Tweet by %s: %s\n", t.User, t.Message)
}
} else {
// No hits
fmt.Print("Found no tweets\n")
}
// Update a tweet by the update API of Elasticsearch.
// We just increment the number of retweets.
script := elastic.NewScript("ctx._source.retweets += num").Param("num", 1)
update, err := client.Update().Index("twitter").Type("tweet").Id("1").
Script(script).
Upsert(map[string]interface{}{"retweets": 0}).
Do()
if err != nil {
// Handle error
panic(err)
}
fmt.Printf("New version of tweet %q is now %d", update.Id, update.Version)
// ...
// Delete an index.
deleteIndex, err := client.DeleteIndex("twitter").Do()
if err != nil {
// Handle error
panic(err)
}
if !deleteIndex.Acknowledged {
// Not acknowledged
}
}
func (t *TestCase) setup() error {
var errorlogger *log.Logger
if t.errorlogfile != "" {
f, err := os.OpenFile(t.errorlogfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0664)
if err != nil {
return err
}
errorlogger = log.New(f, "", log.Ltime|log.Lmicroseconds|log.Lshortfile)
}
var infologger *log.Logger
if t.infologfile != "" {
f, err := os.OpenFile(t.infologfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0664)
if err != nil {
return err
}
infologger = log.New(f, "", log.LstdFlags)
}
// Trace request and response details like this
var tracelogger *log.Logger
if t.tracelogfile != "" {
f, err := os.OpenFile(t.tracelogfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0664)
if err != nil {
return err
}
tracelogger = log.New(f, "", log.LstdFlags)
}
client, err := elastic.NewClient(
elastic.SetURL(t.nodes...),
elastic.SetErrorLog(errorlogger),
elastic.SetInfoLog(infologger),
elastic.SetTraceLog(tracelogger),
elastic.SetMaxRetries(t.maxRetries),
elastic.SetSniff(t.sniff),
elastic.SetSnifferInterval(t.snifferInterval),
elastic.SetHealthcheck(t.healthcheck),
elastic.SetHealthcheckInterval(t.healthcheckInterval))
if err != nil {
// Handle error
return err
}
t.client = client
// Use the IndexExists service to check if a specified index exists.
exists, err := t.client.IndexExists(t.index).Do()
if err != nil {
return err
}
if exists {
deleteIndex, err := t.client.DeleteIndex(t.index).Do()
if err != nil {
return err
}
if !deleteIndex.Acknowledged {
return errors.New("delete index not acknowledged")
}
}
// Create a new index.
createIndex, err := t.client.CreateIndex(t.index).Do()
if err != nil {
return err
}
if !createIndex.Acknowledged {
return errors.New("create index not acknowledged")
}
// Index a tweet (using JSON serialization)
tweet1 := Tweet{User: "******", Message: "Take Five", Retweets: 0}
_, err = t.client.Index().
Index(t.index).
Type("tweet").
Id("1").
BodyJson(tweet1).
Do()
if err != nil {
return err
}
// Index a second tweet (by string)
tweet2 := `{"user" : "olivere", "message" : "It's a Raggy Waltz"}`
_, err = t.client.Index().
Index(t.index).
Type("tweet").
Id("2").
BodyString(tweet2).
Do()
if err != nil {
return err
}
// Flush to make sure the documents got written.
_, err = t.client.Flush().Index(t.index).Do()
if err != nil {
return err
}
return nil
}
func main() {
if len(os.Args) > 1 {
switch os.Args[1] {
case "showenv":
log.Printf("Environment")
env := os.Environ()
sort.Strings(env)
for _, e := range env {
log.Printf("- %s", e)
}
os.Exit(0)
}
}
// Give Elasticsearch some time to startup
time.Sleep(10 * time.Second)
log.SetFlags(log.LstdFlags | log.Lshortfile)
ips, err := net.LookupIP("elasticsearch")
if err != nil {
log.Fatal(err)
}
log.Printf("Lookup for elasticsearch returns the following IPs:")
for _, ip := range ips {
log.Printf("%v", ip)
}
log.Printf("Retrieving http://elasticsearch:9200:")
res, err := http.Get("http://elasticsearch:9200")
if err != nil {
log.Fatal(err)
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
log.Fatal(err)
}
log.Printf("%v", string(body))
url := "http://elasticsearch:9200"
log.Printf("connecting to %v", url)
client, err := elastic.NewClient(elastic.SetURL(url))
if err != nil {
log.Fatal(err)
}
log.Printf("connected to %v", url)
errc := make(chan error)
go func() {
err := showNodes(client)
if err != nil {
log.Printf("nodes info failed: %v", err)
}
t := time.NewTicker(10 * time.Second)
for {
select {
case <-t.C:
err := showNodes(client)
if err != nil {
log.Printf("nodes info failed: %v", err)
}
}
}
}()
go func() {
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, os.Kill)
log.Printf("existing with signal %v", fmt.Sprint(<-c))
errc <- nil
}()
if err := <-errc; err != nil {
os.Exit(1)
}
}
func ExampleScrollService() {
client, err := elastic.NewClient()
if err != nil {
panic(err)
}
// This example illustrates how to use two goroutines to iterate
// through a result set via ScrollService.
//
// It uses the excellent golang.org/x/sync/errgroup package to do so.
//
// The first goroutine will Scroll through the result set and send
// individual results to a channel.
//
// The second goroutine will receive results from the channel and
// deserialize them.
//
// Feel free to add a third goroutine to do something with the
// deserialized results from the 2nd goroutine.
//
// Let's go.
// 1st goroutine sends individual hits to channel.
hits := make(chan json.RawMessage)
g, ctx := errgroup.WithContext(context.Background())
g.Go(func() error {
defer close(hits)
scroll := client.Scroll("twitter").Size(100)
for {
results, err := scroll.Do()
if err == io.EOF {
return nil // all results retrieved
}
if err != nil {
return err // something went wrong
}
// Send the hits to the hits channel
for _, hit := range results.Hits.Hits {
hits <- *hit.Source
}
// Check if we need to terminate early
select {
default:
case <-ctx.Done():
return ctx.Err()
}
}
})
// 2nd goroutine receives hits and deserializes them.
//
// If you want, setup a number of goroutines handling deserialization in parallel.
g.Go(func() error {
for hit := range hits {
// Deserialize
var tw Tweet
err := json.Unmarshal(hit, &tw)
if err != nil {
return err
}
// Do something with the tweet here, e.g. send it to another channel
// for further processing.
_ = tw
// Terminate early?
select {
default:
case <-ctx.Done():
return ctx.Err()
}
}
return nil
})
// Check whether any goroutines failed.
if err := g.Wait(); err != nil {
panic(err)
}
// Done.
fmt.Print("Successfully processed tweets in parallel via ScrollService.\n")
}
