func crawl(index *structs.Index, fileNames []string) {
// channel to recieve doc-keyword pairs.
c := make(chan DocKeywordPair)
// channel to count number of go-routines.
w := make(chan string)
semaphore := make(chan interface{}, MAX_FILES_OPEN)
goroutinesCount := len(fileNames)
for _, fileName := range fileNames {
go processFile(fileName, c, w, semaphore)
}
for goroutinesCount > 0 {
select {
case docKeywordPair := <-c:
index.ConnectKeywordDoc(docKeywordPair.Keyword,
docKeywordPair.Doc)
case name := <-w:
goroutinesCount--
percentage := (1.0 - float64(goroutinesCount)/float64(len(fileNames))) * 100
fmt.Printf("Processed file %v\t%6.2f%%\n", name, percentage)
}
}
}
func connect(index *structs.Index, keyword string, doc string) {
index.ConnectKeywordDoc(keyword, doc)
fmt.Printf("%q and %q were connected in the index\n", keyword, doc)
}