func doIndex() bool {
idxSegm, err := gcse.IndexSegments.GenMaxSegment()
if err != nil {
log.Printf("GenMaxSegment failed: %v", err)
return false
}
runtime.GC()
gcse.DumpMemStats()
log.Printf("Indexing to %v ...", idxSegm)
fpDocDB := sophie.LocalFsPath(configs.DocsDBPath().S())
ts, err := gcse.Index(kv.DirInput(fpDocDB), idxSegm.Join("").S())
if err != nil {
log.Printf("Indexing failed: %v", err)
return false
}
if !func() bool {
f, err := idxSegm.Join(gcse.IndexFn).Create()
if err != nil {
log.Printf("Create index file failed: %v", err)
return false
}
defer f.Close()
log.Printf("Saving index to %v ...", idxSegm)
if err := ts.Save(f); err != nil {
log.Printf("ts.Save failed: %v", err)
return false
}
return true
}() {
return false
}
runtime.GC()
gcse.DumpMemStats()
storePath := idxSegm.Join(configs.FnStore)
log.Printf("Saving store snapshot to %v", storePath)
if err := store.SaveSnapshot(storePath.S()); err != nil {
log.Printf("SaveSnapshot %v failed: %v", storePath, err)
}
if err := idxSegm.Done(); err != nil {
log.Printf("segm.Done failed: %v", err)
return false
}
log.Printf("Indexing success: %s (%d)", idxSegm, ts.DocCount())
gcse.AddBiValueAndProcess(bi.Average, "index.doc-count", ts.DocCount())
ts = nil
gcse.DumpMemStats()
runtime.GC()
gcse.DumpMemStats()
return true
}
func syncDatabases() {
gcse.DumpMemStats()
log.Printf("Synchronizing databases to disk...")
if err := cDB.Sync(); err != nil {
log.Fatalf("cdb.Sync() failed: %v", err)
}
gcse.DumpMemStats()
runtime.GC()
gcse.DumpMemStats()
}
func doIndex() bool {
idxSegm, err := gcse.IndexSegments.GenMaxSegment()
if err != nil {
log.Printf("GenMaxSegment failed: %v", err)
return false
}
runtime.GC()
gcse.DumpMemStats()
log.Printf("Indexing to %v ...", idxSegm)
fpDocDB := sophie.LocalFsPath(gcse.DocsDBPath.S())
ts, err := gcse.Index(kv.DirInput(fpDocDB))
if err != nil {
log.Printf("Indexing failed: %v", err)
return false
}
f, err := idxSegm.Join(gcse.IndexFn).Create()
if err != nil {
log.Printf("Create index file failed: %v", err)
return false
}
//defer f.Close()
log.Printf("Saving index to %v ...", idxSegm)
if err := ts.Save(f); err != nil {
log.Printf("ts.Save failed: %v", err)
return false
}
f.Close()
f = nil
runtime.GC()
gcse.DumpMemStats()
if err := idxSegm.Done(); err != nil {
log.Printf("segm.Done failed: %v", err)
return false
}
log.Printf("Indexing success: %s (%d)", idxSegm, ts.DocCount())
ts = nil
gcse.DumpMemStats()
runtime.GC()
gcse.DumpMemStats()
return true
}
func doIndex(dbSegm gcse.Segment) bool {
idxSegm, err := gcse.IndexSegments.GenMaxSegment()
if err != nil {
log.Printf("GenMaxSegment failed: %v", err)
return false
}
runtime.GC()
gcse.DumpMemStats()
log.Printf("Reading docDB from %v ...", dbSegm)
// read docDB
docDB := gcse.PackedDocDB{gcse.NewMemDB(dbSegm.Join(""), gcse.KindDocDB)}
log.Printf("Indexing to %v ...", idxSegm)
ts, err := gcse.Index(docDB)
if err != nil {
log.Printf("Indexing failed: %v", err)
return false
}
f, err := idxSegm.Join(gcse.IndexFn).Create()
if err != nil {
log.Printf("Create index file failed: %v", err)
return false
}
defer f.Close()
if err := ts.Save(f); err != nil {
log.Printf("ts.Save failed: %v", err)
return false
}
if err := idxSegm.Done(); err != nil {
log.Printf("segm.Done failed: %v", err)
return false
}
log.Printf("Indexing success: %s (%d)", idxSegm, ts.DocCount())
docDB.MemDB, ts = nil, nil
gcse.DumpMemStats()
runtime.GC()
gcse.DumpMemStats()
if err := dbSegm.Remove(); err != nil {
log.Printf("Delete segment %v failed: %v", dbSegm, err)
}
return true
}
func loadIndex() error {
segm, err := gcse.IndexSegments.FindMaxDone()
if segm == nil || err != nil {
return err
}
if indexSegment != nil && !gcse.SegmentLess(indexSegment, segm) {
// no new index
return nil
}
db := &index.TokenSetSearcher{}
f, err := segm.Join(gcse.IndexFn).Open()
if err != nil {
return err
}
defer f.Close()
if err := db.Load(f); err != nil {
return err
}
indexSegment = segm
log.Printf("Load index from %v (%d packages)", segm, db.DocCount())
indexDBBox.Set(db)
updateTime := time.Now()
if st, err := segm.Join(gcse.IndexFn).Stat(); err == nil {
updateTime = st.ModTime()
}
gIndexUpdated = updateTime
var projects villa.StrSet
db.Search(nil, func(docID int32, data interface{}) error {
hit := data.(gcse.HitInfo)
projects.Put(hit.ProjectURL)
return nil
})
gProjectCount = len(projects)
db = nil
gcse.DumpMemStats()
runtime.GC()
gcse.DumpMemStats()
return nil
}
func syncDatabases() {
gcse.DumpMemStats()
log.Printf("Synchronizing databases to disk...")
if err := docDB.Sync(); err != nil {
log.Printf("docDB.Sync failed: %v", err)
}
if err := cPackageDB.Sync(); err != nil {
log.Printf("cPackageDB.Sync failed: %v", err)
}
if err := cPersonDB.Sync(); err != nil {
log.Printf("cPersonDB.Sync failed: %v", err)
}
gcse.DumpMemStats()
runtime.GC()
gcse.DumpMemStats()
}
func loadIndex() error {
segm, err := gcse.IndexSegments.FindMaxDone()
if segm == nil || err != nil {
return err
}
if indexSegment != nil && !gcse.SegmentLess(indexSegment, segm) {
// no new index
return nil
}
db := &index.TokenSetSearcher{}
f, err := segm.Join(gcse.IndexFn).Open()
if err != nil {
return err
}
defer f.Close()
if err := db.Load(f); err != nil {
return err
}
indexSegment = segm
log.Printf("Load index from %v (%d packages)", segm, db.DocCount())
indexDBBox.Set(db)
updateTime := time.Now()
if st, err := segm.Join(gcse.IndexFn).Stat(); err == nil {
updateTime = st.ModTime()
}
indexUpdated = updateTime
db = nil
gcse.DumpMemStats()
runtime.GC()
gcse.DumpMemStats()
return nil
}
func dumpingStatusLoop() {
for time.Now().Before(AppStopTime) {
gcse.DumpMemStats()
time.Sleep(10 * time.Minute)
}
}
func loadIndex() error {
segm, err := gcse.IndexSegments.FindMaxDone()
if segm == nil || err != nil {
return err
}
if indexSegment != nil && !gcse.SegmentLess(indexSegment, segm) {
// no new index
return nil
}
db := &searcherDB{}
if err := func() error {
f, err := segm.Join(gcse.IndexFn).Open()
if err != nil {
return err
}
defer f.Close()
return db.ts.Load(f)
}(); err != nil {
return err
}
db.storeDB = &bh.RefCountBox{
DataPath: func() string {
return segm.Join(configs.FnStore).S()
},
}
hitsPath := segm.Join(gcse.HitsArrFn)
if db.hits, err = index.OpenConstArray(hitsPath.S()); err != nil {
log.Printf("OpenConstArray %v failed: %v", hitsPath, err)
return err
}
// Calculate db.projectCount
var projects stringsp.Set
db.ts.Search(nil, func(docID int32, data interface{}) error {
hit := data.(gcse.HitInfo)
projects.Add(hit.ProjectURL)
return nil
})
db.projectCount = len(projects)
gcse.AddBiValueAndProcess(bi.Max, "index.proj-count", db.projectCount)
// Update db.indexUpdated
db.indexUpdated = time.Now()
if st, err := segm.Join(gcse.IndexFn).Stat(); err == nil {
db.indexUpdated = st.ModTime()
}
indexSegment = segm
log.Printf("Load index from %v (%d packages)", segm, db.PackageCount())
// Exchange new/old database and close the old one.
oldDB := getDatabase()
databaseValue.Store(db)
oldDB.Close()
oldDB = nil
gcse.DumpMemStats()
runtime.GC()
gcse.DumpMemStats()
return nil
}