Exemple #1
0
func TestAEnglish() {
	fmt.Println("** English Benchmark of cloudflare/ahocorasick **")
	fmt.Println("-------------------------------------------------")
	fmt.Println("=> Start to Load... ")
	start := time.Now()
	dict, err := ReadBytes(ENG_DICT_FILE)
	if err != nil {
		fmt.Println(err)
		return
	}

	content, err := ioutil.ReadFile(ENG_TEXT_FILE)
	if err != nil {
		fmt.Println(err)
		return
	}
	end := time.Now()
	fmt.Printf("load file cost:%d(ms)\n", (end.UnixNano()-start.UnixNano())/(1000*1000))

	fmt.Println("=> Start to Search... ")
	start = time.Now()
	m := ahocorasick.NewMatcher(dict)

	//res := m.Match(content)
	m.Match(content)
	end = time.Now()

	fmt.Printf("search cost:%d(ms)\n", (end.UnixNano()-start.UnixNano())/(1000*1000))

	/*
		for _, v := range res {
			fmt.Printf("%d\n", v)
		}
	*/
}
Exemple #2
0
func (this *Confilter) InitMatchers() error {
	matchers := make(map[string]*ahocorasick.Matcher)
	for k, path := range this.config.Dictionaries {
		dictContent, err := ioutil.ReadFile(path)
		if nil != err {
			return err
		}
		dictContentSplited := bytes.Split(dictContent, []byte("\n"))
		matcher := ahocorasick.NewMatcher(dictContentSplited)
		matchers[k] = matcher
	}

	needDoReplace := true

	if len(this.matchers) > 0 {
		if len(matchers) == 0 {
			//when there are running matchers,it will not be replace the well running matcher to empty
			needDoReplace = false
		}
	}

	if needDoReplace {
		this.matchers = matchers
	}

	return nil
}
Exemple #3
0
func initAho() {

	f, err := os.Open("domains.txt")
	if err != nil {
		log.Fatalf("err=%v", err)
	}

	scanner := bufio.NewScanner(f)
	var arr [][]byte
	for scanner.Scan() {
		b := append([]byte(nil), scanner.Bytes()...)
		arr = append(arr, b)
	}

	lookupAho = ahocorasick.NewMatcher(arr)

}