func listAll(index *structs.Index) {
	fmt.Println("Contents of the index:")
	for doc := range index.IteratorDocs() {
		fmt.Printf("%v\n", doc)
		for keyword := range index.IteratorKeywordsByDoc(doc) {
			fmt.Printf("\t%v\n", keyword)
		}
	}
}
func listAllDocs(index *structs.Index) {
	fmt.Println("There are next docs in the index:")
	count := 0
	for doc := range index.IteratorDocs() {
		count++
		fmt.Printf("\t%v\n", doc)
	}
	fmt.Printf("%v in total\n", count)
}
func listAllKeywords(index *structs.Index) {
	fmt.Println("There are next keywords in the index:")
	count := 0
	for keyword := range index.IteratorKeywords() {
		count++
		fmt.Printf("\t%v\n", keyword)
	}
	fmt.Printf("%v in total\n", count)
}
func listKeywords(index *structs.Index, doc string) {
	fmt.Printf("Keywords contained in %q:\n", doc)
	count := 0
	for keyword := range index.IteratorKeywordsByDoc(doc) {
		count++
		fmt.Printf("\t%v\n", keyword)
	}
	fmt.Printf("%v in total\n", count)
}
func listDocs(index *structs.Index, keyword string) {
	fmt.Printf("Documents which contain keyword %q:\n", keyword)
	count := 0
	for doc := range index.IteratorDocsByKeyword(keyword) {
		count++
		fmt.Printf("\t%v\n", doc)
	}
	fmt.Printf("%v in total\n", count)
}
func queryIndex(index *structs.Index, rexpstr string) {
	r, err := regexp.Compile(rexpstr)
	if err != nil {
		fmt.Printf("%v\n", err.Error())
		return
	}
	matchedDocsCount, matchedKeywordsCount := 0, 0
	fmt.Println("Docs matched by regexp:")
	for doc := range index.IteratorDocs() {
		docStr := doc.(string)
		if r.MatchString(docStr) {
			matchedDocsCount++
			fmt.Println(docStr)
		}
	}
	fmt.Printf("%v total.\n", matchedDocsCount)
	fmt.Println("Keywords matched by regexp:")
	for keyword := range index.IteratorKeywords() {
		keywordStr := keyword.(string)
		if r.MatchString(keywordStr) {
			fmt.Println(keyword)
			matchedKeywordsCount++
			inDocs := make([]string, 0)
			for doc := range index.IteratorDocsByKeyword(keywordStr) {
				inDocs = append(inDocs, doc.(string))
			}
			fmt.Printf("\tFound in the next %v documents:\n", len(inDocs))
			for _, doc := range inDocs {
				fmt.Printf("\t%v\n", doc)
			}
		}
	}
	fmt.Printf("%v total.\n", matchedKeywordsCount)
}
예제 #7
0
func crawl(index *structs.Index, fileNames []string) {
	// channel to recieve doc-keyword pairs.
	c := make(chan DocKeywordPair)
	// channel to count number of go-routines.
	w := make(chan string)

	semaphore := make(chan interface{}, MAX_FILES_OPEN)

	goroutinesCount := len(fileNames)
	for _, fileName := range fileNames {
		go processFile(fileName, c, w, semaphore)
	}
	for goroutinesCount > 0 {
		select {
		case docKeywordPair := <-c:
			index.ConnectKeywordDoc(docKeywordPair.Keyword,
				docKeywordPair.Doc)
		case name := <-w:
			goroutinesCount--
			percentage := (1.0 - float64(goroutinesCount)/float64(len(fileNames))) * 100
			fmt.Printf("Processed file %v\t%6.2f%%\n", name, percentage)
		}
	}
}
func connect(index *structs.Index, keyword string, doc string) {
	index.ConnectKeywordDoc(keyword, doc)
	fmt.Printf("%q and %q were connected in the index\n", keyword, doc)
}