func batchIndexingWorker(index bleve.Index, workChan chan *Work, start time.Time) {
for {
select {
case work, ok := <-workChan:
if !ok {
return
}
workSize := 1
if work.batch != nil {
err := index.Batch(work.batch)
if err != nil {
log.Fatalf("indexer worker fatal: %v", err)
}
workSize = work.batch.Size()
} else {
err := index.Index(work.id, work.doc)
if err != nil {
log.Fatalf("indexer worker fatal: %v", err)
}
}
elapsedTime := time.Since(start) / time.Millisecond
updatedTotal := atomic.AddUint64(&totalIndexed, uint64(workSize))
atomic.AddUint64(&totalPlainTextIndexed, work.plainTextBytes)
if updatedTotal%uint64(*printCount) == 0 {
log.Printf("%d,%d", updatedTotal, elapsedTime)
}
}
}
}
func indexWorker(index bleve.Index) {
wikiReader, err := blevebench.NewWikiReader(*source)
if err != nil {
log.Fatal(err)
}
defer wikiReader.Close()
i := 0
a, err := wikiReader.Next()
for a != nil && err == nil && i <= *count {
i++
index.Index(strconv.Itoa(i), a)
a, err = wikiReader.Next()
}
if err != nil {
log.Fatalf("reading worker fatal: %v", err)
}
// dump mem stats if requested
if *memprofile != "" {
f, err := os.Create(*memprofile)
if err != nil {
log.Fatal(err)
}
pprof.WriteHeapProfile(f)
}
}
func indexRecipeLink(i bleve.Index, jsonFilePath string) error {
jsonBytes, err := ioutil.ReadFile(jsonFilePath)
if err != nil {
return err
}
fileName := filepath.Base(jsonFilePath)
ext := filepath.Ext(fileName)
docId := fileName[:(len(fileName) - len(ext))]
return i.Index(docId, jsonBytes)
}
func processUpdate(index bleve.Index, repo *git.Repository, path string) {
log.Printf("updated: %s", path)
rp := relativePath(path)
wiki, err := NewWikiFromFile(path)
if err != nil {
log.Print(err)
} else {
doGitStuff(repo, rp, wiki)
index.Index(rp, wiki)
}
}
func (be *BleveEngine) Index(document *Document) (int64, error) {
start := time.Now().UnixNano() / int64(time.Millisecond)
var index bleve.Index
mapping := bleve.NewIndexMapping()
index, err := bleve.New(INDEX, mapping)
if err != nil {
index, _ = bleve.Open(INDEX)
}
id := (*document).Id
data := (*document).Data
index.Index(id, data)
index.Close()
return time.Now().UnixNano()/int64(time.Millisecond) - start, nil
}
func batchIndexingWorker(index bleve.Index, workChan chan *Work, timeStart time.Time) {
for work := range workChan {
workSize := 1
if work.batch != nil {
err := index.Batch(work.batch)
if err != nil {
log.Fatalf("indexer worker fatal: %v", err)
}
workSize = work.batch.Size()
} else {
err := index.Index(work.id, work.doc)
if err != nil {
log.Fatalf("indexer worker fatal: %v", err)
}
}
atomic.AddUint64(&totalIndexed, uint64(workSize))
atomic.AddUint64(&totalPlainTextIndexed, work.plainTextBytes)
}
}
// Index is used to add the event in the bleve index.
func (e *Event) Index(index bleve.Index) error {
err := index.Index(string(e.ID), e)
return err
}
