func NewSink(raw string) (Sink, error) {
esurl, err := url.Parse(raw)
if err != nil {
return Sink{}, err
}
parts := strings.Split(esurl.Host, ":")
paths := strings.Split(esurl.Path, "/")
if len(parts) == 1 {
return Sink{}, fmt.Errorf("No hostname and port found in `%s`", esurl.Host)
}
if len(paths) < 2 {
return Sink{}, fmt.Errorf("Search index not found in `%s`", esurl.Path)
}
if len(parts) == 2 {
api.Domain = parts[0]
api.Port = parts[1]
} else {
api.Domain = parts[0]
}
return Sink{Index: paths[1], Type: "request", Indexer: core.NewBulkIndexerErrors(10, 60)}, nil
}
// The simplest usage of background bulk indexing with error channel
func ExampleBulkIndexer_errorsmarter() {
indexer := core.NewBulkIndexerErrors(10, 60)
done := make(chan bool)
indexer.Run(done)
errorCt := 0 // use sync.atomic or something if you need
timer := time.NewTicker(time.Minute * 3)
go func() {
for {
select {
case _ = <-timer.C:
if errorCt < 2 {
errorCt = 0
}
case _ = <-done:
return
}
}
}()
go func() {
for errBuf := range indexer.ErrorChannel {
errorCt++
fmt.Println(errBuf.Err)
// log to disk? db? ???? Panic
}
}()
for i := 0; i < 20; i++ {
indexer.Index("twitter", "user", strconv.Itoa(i), "", nil, `{"name":"bob"}`, true)
}
done <- true // send shutdown signal
}
func RebuildSearchIndex(pages []Page) {
indexer := core.NewBulkIndexerErrors(10, 60)
done := make(chan bool)
indexer.Run(done)
go func() {
for errBuf := range indexer.ErrorChannel {
// just blissfully print errors forever
fmt.Println(errBuf.Err)
}
}()
for _, p := range pages {
obj := p.ToSearch()
err := indexer.Index(IndexName, obj.Lang, obj.Id, "", nil, obj, true)
if err != nil {
fmt.Println("err:", err)
}
}
fmt.Println("Finished...")
done <- true
indexer.Flush()
}
// The simplest usage of background bulk indexing
func ExampleBulkIndexer_simple() {
indexer := core.NewBulkIndexerErrors(10, 60)
done := make(chan bool)
indexer.Run(done)
indexer.Index("twitter", "user", "1", "", nil, `{"name":"bob"}`, true)
<-done // wait forever
}
// The simplest usage of background bulk indexing with error channel
func ExampleBulkIndexer_errorchannel() {
indexer := core.NewBulkIndexerErrors(10, 60)
done := make(chan bool)
indexer.Run(done)
go func() {
for errBuf := range indexer.ErrorChannel {
// just blissfully print errors forever
fmt.Println(errBuf.Err)
}
}()
for i := 0; i < 20; i++ {
indexer.Index("twitter", "user", strconv.Itoa(i), "", nil, `{"name":"bob"}`, true)
}
done <- true
}
func NewSink(raw string) (Sink, error) {
esurl, err := url.Parse(raw)
if err != nil {
return Sink{}, err
}
parts := strings.Split(esurl.Host, ":")
paths := strings.Split(esurl.Path, "/")
if len(paths) == 0 || len(parts) == 1 {
return Sink{}, fmt.Errorf("Hostname, port, or index could not be determined")
}
if len(parts) == 2 {
api.Domain = parts[0]
api.Port = parts[1]
} else {
api.Domain = parts[0]
}
return Sink{Index: paths[1], Type: "request", Indexer: core.NewBulkIndexerErrors(10, 60)}, nil
}
func (p *ESPlugin) Init(URI string) {
var err error
err, p.Host, p.ApiPort, p.Index = parseURI(URI)
if err != nil {
log.Fatal("Can't initialize ElasticSearch plugin.", err)
}
api.Domain = p.Host
api.Port = p.ApiPort
p.indexor = core.NewBulkIndexerErrors(50, 60)
p.done = make(chan bool)
p.indexor.Run(p.done)
// Only start the ErrorHandler goroutine when in verbose mode
// no need to burn ressources otherwise
// go p.ErrorHandler()
log.Println("Initialized Elasticsearch Plugin")
return
}