func doIndex() bool {
idxSegm, err := gcse.IndexSegments.GenMaxSegment()
if err != nil {
log.Printf("GenMaxSegment failed: %v", err)
return false
}
runtime.GC()
gcse.DumpMemStats()
log.Printf("Indexing to %v ...", idxSegm)
fpDocDB := sophie.LocalFsPath(configs.DocsDBPath().S())
ts, err := gcse.Index(kv.DirInput(fpDocDB), idxSegm.Join("").S())
if err != nil {
log.Printf("Indexing failed: %v", err)
return false
}
if !func() bool {
f, err := idxSegm.Join(gcse.IndexFn).Create()
if err != nil {
log.Printf("Create index file failed: %v", err)
return false
}
defer f.Close()
log.Printf("Saving index to %v ...", idxSegm)
if err := ts.Save(f); err != nil {
log.Printf("ts.Save failed: %v", err)
return false
}
return true
}() {
return false
}
runtime.GC()
gcse.DumpMemStats()
storePath := idxSegm.Join(configs.FnStore)
log.Printf("Saving store snapshot to %v", storePath)
if err := store.SaveSnapshot(storePath.S()); err != nil {
log.Printf("SaveSnapshot %v failed: %v", storePath, err)
}
if err := idxSegm.Done(); err != nil {
log.Printf("segm.Done failed: %v", err)
return false
}
log.Printf("Indexing success: %s (%d)", idxSegm, ts.DocCount())
gcse.AddBiValueAndProcess(bi.Average, "index.doc-count", ts.DocCount())
ts = nil
gcse.DumpMemStats()
runtime.GC()
gcse.DumpMemStats()
return true
}
func doIndex() bool {
idxSegm, err := gcse.IndexSegments.GenMaxSegment()
if err != nil {
log.Printf("GenMaxSegment failed: %v", err)
return false
}
runtime.GC()
gcse.DumpMemStats()
log.Printf("Indexing to %v ...", idxSegm)
fpDocDB := sophie.LocalFsPath(gcse.DocsDBPath.S())
ts, err := gcse.Index(kv.DirInput(fpDocDB))
if err != nil {
log.Printf("Indexing failed: %v", err)
return false
}
f, err := idxSegm.Join(gcse.IndexFn).Create()
if err != nil {
log.Printf("Create index file failed: %v", err)
return false
}
//defer f.Close()
log.Printf("Saving index to %v ...", idxSegm)
if err := ts.Save(f); err != nil {
log.Printf("ts.Save failed: %v", err)
return false
}
f.Close()
f = nil
runtime.GC()
gcse.DumpMemStats()
if err := idxSegm.Done(); err != nil {
log.Printf("segm.Done failed: %v", err)
return false
}
log.Printf("Indexing success: %s (%d)", idxSegm, ts.DocCount())
ts = nil
gcse.DumpMemStats()
runtime.GC()
gcse.DumpMemStats()
return true
}
func doIndex(dbSegm gcse.Segment) bool {
idxSegm, err := gcse.IndexSegments.GenMaxSegment()
if err != nil {
log.Printf("GenMaxSegment failed: %v", err)
return false
}
runtime.GC()
gcse.DumpMemStats()
log.Printf("Reading docDB from %v ...", dbSegm)
// read docDB
docDB := gcse.PackedDocDB{gcse.NewMemDB(dbSegm.Join(""), gcse.KindDocDB)}
log.Printf("Indexing to %v ...", idxSegm)
ts, err := gcse.Index(docDB)
if err != nil {
log.Printf("Indexing failed: %v", err)
return false
}
f, err := idxSegm.Join(gcse.IndexFn).Create()
if err != nil {
log.Printf("Create index file failed: %v", err)
return false
}
defer f.Close()
if err := ts.Save(f); err != nil {
log.Printf("ts.Save failed: %v", err)
return false
}
if err := idxSegm.Done(); err != nil {
log.Printf("segm.Done failed: %v", err)
return false
}
log.Printf("Indexing success: %s (%d)", idxSegm, ts.DocCount())
docDB.MemDB, ts = nil, nil
gcse.DumpMemStats()
runtime.GC()
gcse.DumpMemStats()
if err := dbSegm.Remove(); err != nil {
log.Printf("Delete segment %v failed: %v", dbSegm, err)
}
return true
}
