func listAll(index *structs.Index) { fmt.Println("Contents of the index:") for doc := range index.IteratorDocs() { fmt.Printf("%v\n", doc) for keyword := range index.IteratorKeywordsByDoc(doc) { fmt.Printf("\t%v\n", keyword) } } }
func listAllDocs(index *structs.Index) { fmt.Println("There are next docs in the index:") count := 0 for doc := range index.IteratorDocs() { count++ fmt.Printf("\t%v\n", doc) } fmt.Printf("%v in total\n", count) }
func listAllKeywords(index *structs.Index) { fmt.Println("There are next keywords in the index:") count := 0 for keyword := range index.IteratorKeywords() { count++ fmt.Printf("\t%v\n", keyword) } fmt.Printf("%v in total\n", count) }
func listKeywords(index *structs.Index, doc string) { fmt.Printf("Keywords contained in %q:\n", doc) count := 0 for keyword := range index.IteratorKeywordsByDoc(doc) { count++ fmt.Printf("\t%v\n", keyword) } fmt.Printf("%v in total\n", count) }
func listDocs(index *structs.Index, keyword string) { fmt.Printf("Documents which contain keyword %q:\n", keyword) count := 0 for doc := range index.IteratorDocsByKeyword(keyword) { count++ fmt.Printf("\t%v\n", doc) } fmt.Printf("%v in total\n", count) }
func queryIndex(index *structs.Index, rexpstr string) { r, err := regexp.Compile(rexpstr) if err != nil { fmt.Printf("%v\n", err.Error()) return } matchedDocsCount, matchedKeywordsCount := 0, 0 fmt.Println("Docs matched by regexp:") for doc := range index.IteratorDocs() { docStr := doc.(string) if r.MatchString(docStr) { matchedDocsCount++ fmt.Println(docStr) } } fmt.Printf("%v total.\n", matchedDocsCount) fmt.Println("Keywords matched by regexp:") for keyword := range index.IteratorKeywords() { keywordStr := keyword.(string) if r.MatchString(keywordStr) { fmt.Println(keyword) matchedKeywordsCount++ inDocs := make([]string, 0) for doc := range index.IteratorDocsByKeyword(keywordStr) { inDocs = append(inDocs, doc.(string)) } fmt.Printf("\tFound in the next %v documents:\n", len(inDocs)) for _, doc := range inDocs { fmt.Printf("\t%v\n", doc) } } } fmt.Printf("%v total.\n", matchedKeywordsCount) }
func crawl(index *structs.Index, fileNames []string) { // channel to recieve doc-keyword pairs. c := make(chan DocKeywordPair) // channel to count number of go-routines. w := make(chan string) semaphore := make(chan interface{}, MAX_FILES_OPEN) goroutinesCount := len(fileNames) for _, fileName := range fileNames { go processFile(fileName, c, w, semaphore) } for goroutinesCount > 0 { select { case docKeywordPair := <-c: index.ConnectKeywordDoc(docKeywordPair.Keyword, docKeywordPair.Doc) case name := <-w: goroutinesCount-- percentage := (1.0 - float64(goroutinesCount)/float64(len(fileNames))) * 100 fmt.Printf("Processed file %v\t%6.2f%%\n", name, percentage) } } }
func connect(index *structs.Index, keyword string, doc string) { index.ConnectKeywordDoc(keyword, doc) fmt.Printf("%q and %q were connected in the index\n", keyword, doc) }