func listAll(index *structs.Index) {
fmt.Println("Contents of the index:")
for doc := range index.IteratorDocs() {
fmt.Printf("%v\n", doc)
for keyword := range index.IteratorKeywordsByDoc(doc) {
fmt.Printf("\t%v\n", keyword)
}
}
}
func listAllDocs(index *structs.Index) {
fmt.Println("There are next docs in the index:")
count := 0
for doc := range index.IteratorDocs() {
count++
fmt.Printf("\t%v\n", doc)
}
fmt.Printf("%v in total\n", count)
}
func listAllKeywords(index *structs.Index) {
fmt.Println("There are next keywords in the index:")
count := 0
for keyword := range index.IteratorKeywords() {
count++
fmt.Printf("\t%v\n", keyword)
}
fmt.Printf("%v in total\n", count)
}
func listKeywords(index *structs.Index, doc string) {
fmt.Printf("Keywords contained in %q:\n", doc)
count := 0
for keyword := range index.IteratorKeywordsByDoc(doc) {
count++
fmt.Printf("\t%v\n", keyword)
}
fmt.Printf("%v in total\n", count)
}
func listDocs(index *structs.Index, keyword string) {
fmt.Printf("Documents which contain keyword %q:\n", keyword)
count := 0
for doc := range index.IteratorDocsByKeyword(keyword) {
count++
fmt.Printf("\t%v\n", doc)
}
fmt.Printf("%v in total\n", count)
}
func queryIndex(index *structs.Index, rexpstr string) {
r, err := regexp.Compile(rexpstr)
if err != nil {
fmt.Printf("%v\n", err.Error())
return
}
matchedDocsCount, matchedKeywordsCount := 0, 0
fmt.Println("Docs matched by regexp:")
for doc := range index.IteratorDocs() {
docStr := doc.(string)
if r.MatchString(docStr) {
matchedDocsCount++
fmt.Println(docStr)
}
}
fmt.Printf("%v total.\n", matchedDocsCount)
fmt.Println("Keywords matched by regexp:")
for keyword := range index.IteratorKeywords() {
keywordStr := keyword.(string)
if r.MatchString(keywordStr) {
fmt.Println(keyword)
matchedKeywordsCount++
inDocs := make([]string, 0)
for doc := range index.IteratorDocsByKeyword(keywordStr) {
inDocs = append(inDocs, doc.(string))
}
fmt.Printf("\tFound in the next %v documents:\n", len(inDocs))
for _, doc := range inDocs {
fmt.Printf("\t%v\n", doc)
}
}
}
fmt.Printf("%v total.\n", matchedKeywordsCount)
}
func crawl(index *structs.Index, fileNames []string) {
// channel to recieve doc-keyword pairs.
c := make(chan DocKeywordPair)
// channel to count number of go-routines.
w := make(chan string)
semaphore := make(chan interface{}, MAX_FILES_OPEN)
goroutinesCount := len(fileNames)
for _, fileName := range fileNames {
go processFile(fileName, c, w, semaphore)
}
for goroutinesCount > 0 {
select {
case docKeywordPair := <-c:
index.ConnectKeywordDoc(docKeywordPair.Keyword,
docKeywordPair.Doc)
case name := <-w:
goroutinesCount--
percentage := (1.0 - float64(goroutinesCount)/float64(len(fileNames))) * 100
fmt.Printf("Processed file %v\t%6.2f%%\n", name, percentage)
}
}
}
func connect(index *structs.Index, keyword string, doc string) {
index.ConnectKeywordDoc(keyword, doc)
fmt.Printf("%q and %q were connected in the index\n", keyword, doc)
}