Esempio n. 1
0
func (c *ChsName) loadNameDict(filePath string, dict map[rune]rune) (err error) {
	err = utils.EachLine(filePath, func(line string) {
		if len(line) > 0 {
			runes := utils.ToRunes(line)
			dict[runes[0]] = runes[0]
		}
	})
	return
}
Esempio n. 2
0
func (s *StopWord) Load(file string) (err error) {
	err = utils.EachLine(file, func(line string) {
		if len(line) > 0 {
			if utils.FirstRune(line) < 128 {
				s.stopWordTbl[strings.ToLower(line)] = true
			} else {
				s.stopWordTbl[line] = true
			}
		}
	})
	return
}
Esempio n. 3
0
func (d *WordDictionary) loadFromTextFile(fileName string) (dicts *list.List, err error) {
	dicts = list.New()
	err = utils.EachLine(fileName, func(line string) {
		words := strings.Split(string(line), "|")
		if len(words) == 3 {
			word := strings.TrimSpace(words[0])
			pos, _ := strconv.ParseInt(words[1], 0, 0)
			frequency, _ := strconv.ParseFloat(words[2], 64)
			dicts.PushBack(NewWordAttr(word, int(pos), frequency))
		}
	})
	return
}
Esempio n. 4
0
func (s *Segment) loadVerbTable(file string) (err error) {
	s.verbTable = make(map[string]string)
	err = utils.EachLine(file, func(line string) {
		words := strings.Split(line, "\t")
		if len(words) == 3 {
			value := strings.TrimSpace(strings.ToLower(words[0]))
			for j := 1; j < 3; j++ {
				key := strings.TrimSpace(strings.ToLower(words[j]))
				s.verbTable[key] = value
			}
		}
	})
	return
}
Esempio n. 5
0
func (s *Synonym) Load(dictPath string) (err error) {
	err = utils.EachLine(dictPath+"/"+SynonymFileName, func(line string) {
		if len(line) > 0 {
			words := strings.Split(line, ",")
			s.groupList = append(s.groupList, words)
			groupId := len(s.groupList) - 1
			for i := 0; i < len(words); i++ {
				key := strings.TrimSpace(words[i])
				if l, ok := s.wordToGroupId[key]; ok {
					if l[len(l)-1] == groupId {
						continue
					}
					s.wordToGroupId[key] = append(s.wordToGroupId[key], groupId)
				} else {
					s.wordToGroupId[key] = []int{groupId}
				}
			}
		}
	})
	return
}