Exemplo n.º 1
0
func NewSink(raw string) (Sink, error) {
	esurl, err := url.Parse(raw)

	if err != nil {
		return Sink{}, err
	}

	parts := strings.Split(esurl.Host, ":")
	paths := strings.Split(esurl.Path, "/")

	if len(parts) == 1 {
		return Sink{}, fmt.Errorf("No hostname and port found in `%s`", esurl.Host)
	}

	if len(paths) < 2 {
		return Sink{}, fmt.Errorf("Search index not found in `%s`", esurl.Path)
	}

	if len(parts) == 2 {
		api.Domain = parts[0]
		api.Port = parts[1]
	} else {
		api.Domain = parts[0]
	}

	return Sink{Index: paths[1], Type: "request", Indexer: core.NewBulkIndexerErrors(10, 60)}, nil
}
Exemplo n.º 2
0
// The simplest usage of background bulk indexing with error channel
func ExampleBulkIndexer_errorsmarter() {
	indexer := core.NewBulkIndexerErrors(10, 60)
	done := make(chan bool)
	indexer.Run(done)

	errorCt := 0 // use sync.atomic or something if you need
	timer := time.NewTicker(time.Minute * 3)
	go func() {
		for {
			select {
			case _ = <-timer.C:
				if errorCt < 2 {
					errorCt = 0
				}
			case _ = <-done:
				return
			}
		}
	}()

	go func() {
		for errBuf := range indexer.ErrorChannel {
			errorCt++
			fmt.Println(errBuf.Err)
			// log to disk?  db?   ????  Panic
		}
	}()
	for i := 0; i < 20; i++ {
		indexer.Index("twitter", "user", strconv.Itoa(i), "", nil, `{"name":"bob"}`, true)
	}
	done <- true // send shutdown signal
}
Exemplo n.º 3
0
func RebuildSearchIndex(pages []Page) {

	indexer := core.NewBulkIndexerErrors(10, 60)
	done := make(chan bool)
	indexer.Run(done)

	go func() {
		for errBuf := range indexer.ErrorChannel {
			// just blissfully print errors forever
			fmt.Println(errBuf.Err)
		}
	}()

	for _, p := range pages {
		obj := p.ToSearch()
		err := indexer.Index(IndexName, obj.Lang, obj.Id, "", nil, obj, true)

		if err != nil {
			fmt.Println("err:", err)
		}
	}

	fmt.Println("Finished...")
	done <- true
	indexer.Flush()
}
Exemplo n.º 4
0
// The simplest usage of background bulk indexing
func ExampleBulkIndexer_simple() {
	indexer := core.NewBulkIndexerErrors(10, 60)
	done := make(chan bool)
	indexer.Run(done)

	indexer.Index("twitter", "user", "1", "", nil, `{"name":"bob"}`, true)

	<-done // wait forever
}
Exemplo n.º 5
0
// The simplest usage of background bulk indexing with error channel
func ExampleBulkIndexer_errorchannel() {
	indexer := core.NewBulkIndexerErrors(10, 60)
	done := make(chan bool)
	indexer.Run(done)

	go func() {
		for errBuf := range indexer.ErrorChannel {
			// just blissfully print errors forever
			fmt.Println(errBuf.Err)
		}
	}()
	for i := 0; i < 20; i++ {
		indexer.Index("twitter", "user", strconv.Itoa(i), "", nil, `{"name":"bob"}`, true)
	}
	done <- true
}
Exemplo n.º 6
0
func NewSink(raw string) (Sink, error) {
	esurl, err := url.Parse(raw)

	if err != nil {
		return Sink{}, err
	}

	parts := strings.Split(esurl.Host, ":")
	paths := strings.Split(esurl.Path, "/")

	if len(paths) == 0 || len(parts) == 1 {
		return Sink{}, fmt.Errorf("Hostname, port, or index could not be determined")
	}

	if len(parts) == 2 {
		api.Domain = parts[0]
		api.Port = parts[1]
	} else {
		api.Domain = parts[0]
	}

	return Sink{Index: paths[1], Type: "request", Indexer: core.NewBulkIndexerErrors(10, 60)}, nil
}
Exemplo n.º 7
0
func (p *ESPlugin) Init(URI string) {
	var err error

	err, p.Host, p.ApiPort, p.Index = parseURI(URI)

	if err != nil {
		log.Fatal("Can't initialize ElasticSearch plugin.", err)
	}

	api.Domain = p.Host
	api.Port = p.ApiPort

	p.indexor = core.NewBulkIndexerErrors(50, 60)
	p.done = make(chan bool)
	p.indexor.Run(p.done)

	// Only start the ErrorHandler goroutine when in verbose mode
	// no need to burn ressources otherwise
	// go p.ErrorHandler()

	log.Println("Initialized Elasticsearch Plugin")
	return
}