func main() { docDB := gcse.NewMemDB(DocDBPath, gcse.KindDocDB) countAll, countReadme, countHasSents := 0, 0, 0 countSents := 0 f, err := villa.Path("exps/notfound.txt").Create() if err != nil { log.Fatal(err) } defer f.Close() log.Printf("Start processing ...") if err := docDB.Iterate(func(key string, val interface{}) error { countAll++ d := val.(gcse.DocInfo) if d.ReadmeData != "" { countReadme++ readme := gcse.ReadmeToText(d.ReadmeFn, d.ReadmeData) sents := gcse.ChooseImportantSentenses(readme, d.Name, d.Package) if len(sents) > 0 { countSents += len(sents) countHasSents++ } else { fmt.Fprintln(f, "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") fmt.Fprintf(f, "%s - %s - %s\n", d.Name, d.Package, d.ReadmeFn) fmt.Fprintf(f, "%s\n", readme) } } return nil }); err != nil { log.Fatalf("docDB.Iterate failed: %v", err) } log.Printf("%d documents processed.", countAll) log.Printf("%d have readme.", countReadme) log.Printf("%d found %d important sentenses.", countHasSents, countSents) }
func showSearchResults(results *SearchResult, tokens stringsp.Set, r Range) *ShowResults { docs := make([]ShowDocInfo, 0, len(results.Hits)) projToIdx := make(map[string]int) folded := 0 cnt := 0 mainLoop: for _, d := range results.Hits { d.Name = packageShowName(d.Name, d.Package) parts := strings.Split(d.Package, "/") if len(parts) > 2 { // try fold it (if its parent has been in the list) for i := len(parts) - 1; i >= 2; i-- { pkg := strings.Join(parts[:i], "/") if idx, ok := projToIdx[pkg]; ok { markedName := markText(d.Name, tokens, markWord) if r.In(idx) { docsIdx := idx - r.start docs[docsIdx].Subs = append(docs[docsIdx].Subs, SubProjectInfo{ MarkedName: markedName, Package: d.Package, SubPath: "/" + strings.Join(parts[i:], "/"), Info: d.Synopsis, }) } folded++ continue mainLoop } } } projToIdx[d.Package] = cnt if r.In(cnt) { markedName := markText(d.Name, tokens, markWord) readme := gcse.ReadmeToText(d.ReadmeFn, d.ReadmeData) if len(readme) > 20*1024 { readme = readme[:20*1024] } desc := d.Description for _, sent := range d.ImportantSentences { desc += "\n" + sent } desc += "\n" + readme raw := selectSnippets(desc, tokens, 300) if d.StarCount < 0 { d.StarCount = 0 } docs = append(docs, ShowDocInfo{ Hit: d, Index: cnt + 1, MarkedName: markedName, Summary: markText(raw, tokens, markWord), MarkedPackage: markText(d.Package, tokens, markWord), }) } cnt++ } return &ShowResults{ TotalResults: results.TotalResults, TotalEntries: cnt, Folded: folded, Docs: docs, } }