// AddToken adds a new word with IDF into it's dictionary. func (i *Idf) AddToken(token dictionary.Token) { i.Lock() i.freqMap[token.Text()] = token.Frequency() i.freqs = append(i.freqs, token.Frequency()) sort.Float64s(i.freqs) i.median = i.freqs[len(i.freqs)/2] i.Unlock() }
func (d *Dictionary) addToken(token dictionary.Token) { d.freqMap[token.Text()] = token.Frequency() d.total += token.Frequency() runes := []rune(token.Text()) n := len(runes) for i := 0; i < n; i++ { //TODO: n-1? frag := string(runes[:i+1]) if _, ok := d.freqMap[frag]; !ok { d.freqMap[frag] = 0.0 } } }
// AddToken adds a token into StopWord dictionary. func (s *StopWord) AddToken(token dictionary.Token) { s.Lock() s.stopWordMap[token.Text()] = 1 s.Unlock() }