func main() {
docDB := gcse.NewMemDB(DocDBPath, gcse.KindDocDB)
countAll, countReadme, countHasSents := 0, 0, 0
countSents := 0
f, err := villa.Path("exps/notfound.txt").Create()
if err != nil {
log.Fatal(err)
}
defer f.Close()
log.Printf("Start processing ...")
if err := docDB.Iterate(func(key string, val interface{}) error {
countAll++
d := val.(gcse.DocInfo)
if d.ReadmeData != "" {
countReadme++
readme := gcse.ReadmeToText(d.ReadmeFn, d.ReadmeData)
sents := gcse.ChooseImportantSentenses(readme, d.Name, d.Package)
if len(sents) > 0 {
countSents += len(sents)
countHasSents++
} else {
fmt.Fprintln(f, "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$")
fmt.Fprintf(f, "%s - %s - %s\n", d.Name, d.Package, d.ReadmeFn)
fmt.Fprintf(f, "%s\n", readme)
}
}
return nil
}); err != nil {
log.Fatalf("docDB.Iterate failed: %v", err)
}
log.Printf("%d documents processed.", countAll)
log.Printf("%d have readme.", countReadme)
log.Printf("%d found %d important sentenses.", countHasSents, countSents)
}
func showSearchResults(results *SearchResult, tokens stringsp.Set,
r Range) *ShowResults {
docs := make([]ShowDocInfo, 0, len(results.Hits))
projToIdx := make(map[string]int)
folded := 0
cnt := 0
mainLoop:
for _, d := range results.Hits {
d.Name = packageShowName(d.Name, d.Package)
parts := strings.Split(d.Package, "/")
if len(parts) > 2 {
// try fold it (if its parent has been in the list)
for i := len(parts) - 1; i >= 2; i-- {
pkg := strings.Join(parts[:i], "/")
if idx, ok := projToIdx[pkg]; ok {
markedName := markText(d.Name, tokens, markWord)
if r.In(idx) {
docsIdx := idx - r.start
docs[docsIdx].Subs = append(docs[docsIdx].Subs,
SubProjectInfo{
MarkedName: markedName,
Package: d.Package,
SubPath: "/" + strings.Join(parts[i:], "/"),
Info: d.Synopsis,
})
}
folded++
continue mainLoop
}
}
}
projToIdx[d.Package] = cnt
if r.In(cnt) {
markedName := markText(d.Name, tokens, markWord)
readme := gcse.ReadmeToText(d.ReadmeFn, d.ReadmeData)
if len(readme) > 20*1024 {
readme = readme[:20*1024]
}
desc := d.Description
for _, sent := range d.ImportantSentences {
desc += "\n" + sent
}
desc += "\n" + readme
raw := selectSnippets(desc, tokens, 300)
if d.StarCount < 0 {
d.StarCount = 0
}
docs = append(docs, ShowDocInfo{
Hit: d,
Index: cnt + 1,
MarkedName: markedName,
Summary: markText(raw, tokens, markWord),
MarkedPackage: markText(d.Package, tokens, markWord),
})
}
cnt++
}
return &ShowResults{
TotalResults: results.TotalResults,
TotalEntries: cnt,
Folded: folded,
Docs: docs,
}
}