// NewIndex creates a new index for the .go files provided by the corpus. func (c *Corpus) NewIndex() *Index { // initialize Indexer // (use some reasonably sized maps to start) x := &Indexer{ c: c, fset: token.NewFileSet(), fsOpenGate: make(chan bool, maxOpenFiles), strings: make(map[string]string), packages: make(map[Pak]*Pak, 256), words: make(map[string]*IndexResult, 8192), throttle: util.NewThrottle(c.throttle(), 100*time.Millisecond), // run at least 0.1s at a time importCount: make(map[string]int), packagePath: make(map[string]map[string]bool), exports: make(map[string]map[string]SpotKind), idents: make(map[SpotKind]map[string][]Ident, 4), } // index all files in the directories given by dirnames var wg sync.WaitGroup // outstanding ReadDir + visitFile dirGate := make(chan bool, maxOpenDirs) for dirname := range c.fsDirnames() { if c.IndexDirectory != nil && !c.IndexDirectory(dirname) { continue } dirGate <- true wg.Add(1) go func(dirname string) { defer func() { <-dirGate }() defer wg.Done() list, err := c.fs.ReadDir(dirname) if err != nil { log.Printf("ReadDir(%q): %v; skipping directory", dirname, err) return // ignore this directory } for _, fi := range list { wg.Add(1) go func(fi os.FileInfo) { defer wg.Done() x.visitFile(dirname, fi) }(fi) } }(dirname) } wg.Wait() if !c.IndexFullText { // the file set, the current file, and the sources are // not needed after indexing if no text index is built - // help GC and clear them x.fset = nil x.sources.Reset() x.current = nil // contains reference to fset! } // for each word, reduce the RunLists into a LookupResult; // also collect the word with its canonical spelling in a // word list for later computation of alternative spellings words := make(map[string]*LookupResult) var wlist RunList for w, h := range x.words { decls := reduce(h.Decls) others := reduce(h.Others) words[w] = &LookupResult{ Decls: decls, Others: others, } wlist = append(wlist, &wordPair{canonical(w), w}) x.throttle.Throttle() } x.stats.Words = len(words) // reduce the word list {canonical(w), w} into // a list of AltWords runs {canonical(w), {w}} alist := wlist.reduce(lessWordPair, newAltWords) // convert alist into a map of alternative spellings alts := make(map[string]*AltWords) for i := 0; i < len(alist); i++ { a := alist[i].(*AltWords) alts[a.Canon] = a } // create text index var suffixes *suffixarray.Index if c.IndexFullText { suffixes = suffixarray.New(x.sources.Bytes()) } for _, idMap := range x.idents { for _, ir := range idMap { sort.Sort(byPackage(ir)) } } return &Index{ fset: x.fset, suffixes: suffixes, words: words, alts: alts, snippets: x.snippets, stats: x.stats, importCount: x.importCount, packagePath: x.packagePath, exports: x.exports, idents: x.idents, opts: indexOptions{ Docs: x.c.IndexDocs, GoCode: x.c.IndexGoCode, FullText: x.c.IndexFullText, MaxResults: x.c.MaxResults, }, } }
// NewIndex creates a new index for the .go files // in the directories given by dirnames. // func NewIndex(c *Corpus, dirnames <-chan string, fulltextIndex bool, throttle float64) *Index { var x Indexer th := util.NewThrottle(throttle, 100*time.Millisecond) // run at least 0.1s at a time // initialize Indexer // (use some reasonably sized maps to start) x.c = c x.fset = token.NewFileSet() x.packages = make(map[string]*Pak, 256) x.words = make(map[string]*IndexResult, 8192) // index all files in the directories given by dirnames for dirname := range dirnames { list, err := c.fs.ReadDir(dirname) if err != nil { continue // ignore this directory } for _, f := range list { if !f.IsDir() { x.visitFile(dirname, f, fulltextIndex) } th.Throttle() } } if !fulltextIndex { // the file set, the current file, and the sources are // not needed after indexing if no text index is built - // help GC and clear them x.fset = nil x.sources.Reset() x.current = nil // contains reference to fset! } // for each word, reduce the RunLists into a LookupResult; // also collect the word with its canonical spelling in a // word list for later computation of alternative spellings words := make(map[string]*LookupResult) var wlist RunList for w, h := range x.words { decls := reduce(h.Decls) others := reduce(h.Others) words[w] = &LookupResult{ Decls: decls, Others: others, } wlist = append(wlist, &wordPair{canonical(w), w}) th.Throttle() } x.stats.Words = len(words) // reduce the word list {canonical(w), w} into // a list of AltWords runs {canonical(w), {w}} alist := wlist.reduce(lessWordPair, newAltWords) // convert alist into a map of alternative spellings alts := make(map[string]*AltWords) for i := 0; i < len(alist); i++ { a := alist[i].(*AltWords) alts[a.Canon] = a } // create text index var suffixes *suffixarray.Index if fulltextIndex { suffixes = suffixarray.New(x.sources.Bytes()) } return &Index{x.fset, suffixes, words, alts, x.snippets, x.stats} }