Example #1
0
func main() {
	docDB := gcse.NewMemDB(DocDBPath, gcse.KindDocDB)
	countAll, countReadme, countHasSents := 0, 0, 0
	countSents := 0

	f, err := villa.Path("exps/notfound.txt").Create()
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	log.Printf("Start processing ...")
	if err := docDB.Iterate(func(key string, val interface{}) error {
		countAll++

		d := val.(gcse.DocInfo)
		if d.ReadmeData != "" {
			countReadme++

			readme := gcse.ReadmeToText(d.ReadmeFn, d.ReadmeData)

			sents := gcse.ChooseImportantSentenses(readme, d.Name, d.Package)
			if len(sents) > 0 {
				countSents += len(sents)
				countHasSents++
			} else {
				fmt.Fprintln(f, "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$")
				fmt.Fprintf(f, "%s - %s - %s\n", d.Name, d.Package, d.ReadmeFn)
				fmt.Fprintf(f, "%s\n", readme)
			}
		}

		return nil
	}); err != nil {
		log.Fatalf("docDB.Iterate failed: %v", err)
	}

	log.Printf("%d documents processed.", countAll)
	log.Printf("%d have readme.", countReadme)
	log.Printf("%d found %d important sentenses.", countHasSents, countSents)
}
Example #2
0
func showSearchResults(results *SearchResult, tokens stringsp.Set,
	r Range) *ShowResults {
	docs := make([]ShowDocInfo, 0, len(results.Hits))

	projToIdx := make(map[string]int)
	folded := 0

	cnt := 0
mainLoop:
	for _, d := range results.Hits {
		d.Name = packageShowName(d.Name, d.Package)

		parts := strings.Split(d.Package, "/")
		if len(parts) > 2 {
			// try fold it (if its parent has been in the list)
			for i := len(parts) - 1; i >= 2; i-- {
				pkg := strings.Join(parts[:i], "/")
				if idx, ok := projToIdx[pkg]; ok {
					markedName := markText(d.Name, tokens, markWord)
					if r.In(idx) {
						docsIdx := idx - r.start
						docs[docsIdx].Subs = append(docs[docsIdx].Subs,
							SubProjectInfo{
								MarkedName: markedName,
								Package:    d.Package,
								SubPath:    "/" + strings.Join(parts[i:], "/"),
								Info:       d.Synopsis,
							})
					}
					folded++
					continue mainLoop
				}
			}
		}

		projToIdx[d.Package] = cnt
		if r.In(cnt) {
			markedName := markText(d.Name, tokens, markWord)
			readme := gcse.ReadmeToText(d.ReadmeFn, d.ReadmeData)
			if len(readme) > 20*1024 {
				readme = readme[:20*1024]
			}
			desc := d.Description
			for _, sent := range d.ImportantSentences {
				desc += "\n" + sent
			}
			desc += "\n" + readme
			raw := selectSnippets(desc, tokens, 300)

			if d.StarCount < 0 {
				d.StarCount = 0
			}
			docs = append(docs, ShowDocInfo{
				Hit:           d,
				Index:         cnt + 1,
				MarkedName:    markedName,
				Summary:       markText(raw, tokens, markWord),
				MarkedPackage: markText(d.Package, tokens, markWord),
			})
		}
		cnt++
	}

	return &ShowResults{
		TotalResults: results.TotalResults,
		TotalEntries: cnt,
		Folded:       folded,
		Docs:         docs,
	}
}