func TestVulgarWords(t *testing.T) {
for _, td := range dataVulgarWords {
got := tekstus.StringFrequenciesOf(td.text,
tekstus.VulgarWords, false)
assert(t, td.exp, got, true)
}
}
/*
Compute frequency vulgar words in inserted text.
*/
func (ftr *WordsVulgarFrequency) Compute(dataset tabula.DatasetInterface) {
col := dataset.GetColumnByName("additions")
for _, rec := range col.Records {
s := clean.WikiText(rec.String())
freq := tekstus.StringFrequenciesOf(s, tekstus.VulgarWords,
false)
ftr.PushBack(tabula.NewRecordReal(Round(freq)))
}
}
/*
Compute frequency of biased words.
*/
func (ftr *WordsBiasFrequency) Compute(dataset tabula.DatasetInterface) {
col := dataset.GetColumnByName("additions")
for _, rec := range col.Records {
r := tabula.NewRecordReal(float64(0))
text := rec.String()
if len(text) == 0 {
ftr.PushBack(r)
continue
}
in := clean.WikiText(text)
freq := tekstus.StringFrequenciesOf(in,
tekstus.BiasedWords, false)
r.SetFloat(freq)
ftr.PushBack(r)
}
}
