// NewIndex creates a new index for the .go files provided by the corpus.
func (c *Corpus) NewIndex() *Index {
// initialize Indexer
// (use some reasonably sized maps to start)
x := &Indexer{
c: c,
fset: token.NewFileSet(),
fsOpenGate: make(chan bool, maxOpenFiles),
strings: make(map[string]string),
packages: make(map[Pak]*Pak, 256),
words: make(map[string]*IndexResult, 8192),
throttle: util.NewThrottle(c.throttle(), 100*time.Millisecond), // run at least 0.1s at a time
importCount: make(map[string]int),
packagePath: make(map[string]map[string]bool),
exports: make(map[string]map[string]SpotKind),
idents: make(map[SpotKind]map[string][]Ident, 4),
}
// index all files in the directories given by dirnames
var wg sync.WaitGroup // outstanding ReadDir + visitFile
dirGate := make(chan bool, maxOpenDirs)
for dirname := range c.fsDirnames() {
if c.IndexDirectory != nil && !c.IndexDirectory(dirname) {
continue
}
dirGate <- true
wg.Add(1)
go func(dirname string) {
defer func() { <-dirGate }()
defer wg.Done()
list, err := c.fs.ReadDir(dirname)
if err != nil {
log.Printf("ReadDir(%q): %v; skipping directory", dirname, err)
return // ignore this directory
}
for _, fi := range list {
wg.Add(1)
go func(fi os.FileInfo) {
defer wg.Done()
x.visitFile(dirname, fi)
}(fi)
}
}(dirname)
}
wg.Wait()
if !c.IndexFullText {
// the file set, the current file, and the sources are
// not needed after indexing if no text index is built -
// help GC and clear them
x.fset = nil
x.sources.Reset()
x.current = nil // contains reference to fset!
}
// for each word, reduce the RunLists into a LookupResult;
// also collect the word with its canonical spelling in a
// word list for later computation of alternative spellings
words := make(map[string]*LookupResult)
var wlist RunList
for w, h := range x.words {
decls := reduce(h.Decls)
others := reduce(h.Others)
words[w] = &LookupResult{
Decls: decls,
Others: others,
}
wlist = append(wlist, &wordPair{canonical(w), w})
x.throttle.Throttle()
}
x.stats.Words = len(words)
// reduce the word list {canonical(w), w} into
// a list of AltWords runs {canonical(w), {w}}
alist := wlist.reduce(lessWordPair, newAltWords)
// convert alist into a map of alternative spellings
alts := make(map[string]*AltWords)
for i := 0; i < len(alist); i++ {
a := alist[i].(*AltWords)
alts[a.Canon] = a
}
// create text index
var suffixes *suffixarray.Index
if c.IndexFullText {
suffixes = suffixarray.New(x.sources.Bytes())
}
for _, idMap := range x.idents {
for _, ir := range idMap {
sort.Sort(byPackage(ir))
}
}
return &Index{
fset: x.fset,
suffixes: suffixes,
words: words,
alts: alts,
snippets: x.snippets,
stats: x.stats,
importCount: x.importCount,
packagePath: x.packagePath,
exports: x.exports,
idents: x.idents,
opts: indexOptions{
Docs: x.c.IndexDocs,
GoCode: x.c.IndexGoCode,
FullText: x.c.IndexFullText,
MaxResults: x.c.MaxResults,
},
}
}
// NewIndex creates a new index for the .go files
// in the directories given by dirnames.
//
func NewIndex(c *Corpus, dirnames <-chan string, fulltextIndex bool, throttle float64) *Index {
var x Indexer
th := util.NewThrottle(throttle, 100*time.Millisecond) // run at least 0.1s at a time
// initialize Indexer
// (use some reasonably sized maps to start)
x.c = c
x.fset = token.NewFileSet()
x.packages = make(map[string]*Pak, 256)
x.words = make(map[string]*IndexResult, 8192)
// index all files in the directories given by dirnames
for dirname := range dirnames {
list, err := c.fs.ReadDir(dirname)
if err != nil {
continue // ignore this directory
}
for _, f := range list {
if !f.IsDir() {
x.visitFile(dirname, f, fulltextIndex)
}
th.Throttle()
}
}
if !fulltextIndex {
// the file set, the current file, and the sources are
// not needed after indexing if no text index is built -
// help GC and clear them
x.fset = nil
x.sources.Reset()
x.current = nil // contains reference to fset!
}
// for each word, reduce the RunLists into a LookupResult;
// also collect the word with its canonical spelling in a
// word list for later computation of alternative spellings
words := make(map[string]*LookupResult)
var wlist RunList
for w, h := range x.words {
decls := reduce(h.Decls)
others := reduce(h.Others)
words[w] = &LookupResult{
Decls: decls,
Others: others,
}
wlist = append(wlist, &wordPair{canonical(w), w})
th.Throttle()
}
x.stats.Words = len(words)
// reduce the word list {canonical(w), w} into
// a list of AltWords runs {canonical(w), {w}}
alist := wlist.reduce(lessWordPair, newAltWords)
// convert alist into a map of alternative spellings
alts := make(map[string]*AltWords)
for i := 0; i < len(alist); i++ {
a := alist[i].(*AltWords)
alts[a.Canon] = a
}
// create text index
var suffixes *suffixarray.Index
if fulltextIndex {
suffixes = suffixarray.New(x.sources.Bytes())
}
return &Index{x.fset, suffixes, words, alts, x.snippets, x.stats}
}