func TestAEnglish() {
fmt.Println("** English Benchmark of cloudflare/ahocorasick **")
fmt.Println("-------------------------------------------------")
fmt.Println("=> Start to Load... ")
start := time.Now()
dict, err := ReadBytes(ENG_DICT_FILE)
if err != nil {
fmt.Println(err)
return
}
content, err := ioutil.ReadFile(ENG_TEXT_FILE)
if err != nil {
fmt.Println(err)
return
}
end := time.Now()
fmt.Printf("load file cost:%d(ms)\n", (end.UnixNano()-start.UnixNano())/(1000*1000))
fmt.Println("=> Start to Search... ")
start = time.Now()
m := ahocorasick.NewMatcher(dict)
//res := m.Match(content)
m.Match(content)
end = time.Now()
fmt.Printf("search cost:%d(ms)\n", (end.UnixNano()-start.UnixNano())/(1000*1000))
/*
for _, v := range res {
fmt.Printf("%d\n", v)
}
*/
}
func (this *Confilter) InitMatchers() error {
matchers := make(map[string]*ahocorasick.Matcher)
for k, path := range this.config.Dictionaries {
dictContent, err := ioutil.ReadFile(path)
if nil != err {
return err
}
dictContentSplited := bytes.Split(dictContent, []byte("\n"))
matcher := ahocorasick.NewMatcher(dictContentSplited)
matchers[k] = matcher
}
needDoReplace := true
if len(this.matchers) > 0 {
if len(matchers) == 0 {
//when there are running matchers,it will not be replace the well running matcher to empty
needDoReplace = false
}
}
if needDoReplace {
this.matchers = matchers
}
return nil
}
func initAho() {
f, err := os.Open("domains.txt")
if err != nil {
log.Fatalf("err=%v", err)
}
scanner := bufio.NewScanner(f)
var arr [][]byte
for scanner.Scan() {
b := append([]byte(nil), scanner.Bytes()...)
arr = append(arr, b)
}
lookupAho = ahocorasick.NewMatcher(arr)
}