func (c *ChsName) loadNameDict(filePath string, dict map[rune]rune) (err error) {
err = utils.EachLine(filePath, func(line string) {
if len(line) > 0 {
runes := utils.ToRunes(line)
dict[runes[0]] = runes[0]
}
})
return
}
func (s *StopWord) Load(file string) (err error) {
err = utils.EachLine(file, func(line string) {
if len(line) > 0 {
if utils.FirstRune(line) < 128 {
s.stopWordTbl[strings.ToLower(line)] = true
} else {
s.stopWordTbl[line] = true
}
}
})
return
}
func (d *WordDictionary) loadFromTextFile(fileName string) (dicts *list.List, err error) {
dicts = list.New()
err = utils.EachLine(fileName, func(line string) {
words := strings.Split(string(line), "|")
if len(words) == 3 {
word := strings.TrimSpace(words[0])
pos, _ := strconv.ParseInt(words[1], 0, 0)
frequency, _ := strconv.ParseFloat(words[2], 64)
dicts.PushBack(NewWordAttr(word, int(pos), frequency))
}
})
return
}
func (s *Segment) loadVerbTable(file string) (err error) {
s.verbTable = make(map[string]string)
err = utils.EachLine(file, func(line string) {
words := strings.Split(line, "\t")
if len(words) == 3 {
value := strings.TrimSpace(strings.ToLower(words[0]))
for j := 1; j < 3; j++ {
key := strings.TrimSpace(strings.ToLower(words[j]))
s.verbTable[key] = value
}
}
})
return
}
func (s *Synonym) Load(dictPath string) (err error) {
err = utils.EachLine(dictPath+"/"+SynonymFileName, func(line string) {
if len(line) > 0 {
words := strings.Split(line, ",")
s.groupList = append(s.groupList, words)
groupId := len(s.groupList) - 1
for i := 0; i < len(words); i++ {
key := strings.TrimSpace(words[i])
if l, ok := s.wordToGroupId[key]; ok {
if l[len(l)-1] == groupId {
continue
}
s.wordToGroupId[key] = append(s.wordToGroupId[key], groupId)
} else {
s.wordToGroupId[key] = []int{groupId}
}
}
}
})
return
}