func TestVulgarWords(t *testing.T) { for _, td := range dataVulgarWords { got := tekstus.StringFrequenciesOf(td.text, tekstus.VulgarWords, false) assert(t, td.exp, got, true) } }
/* Compute frequency vulgar words in inserted text. */ func (ftr *WordsVulgarFrequency) Compute(dataset tabula.DatasetInterface) { col := dataset.GetColumnByName("additions") for _, rec := range col.Records { s := clean.WikiText(rec.String()) freq := tekstus.StringFrequenciesOf(s, tekstus.VulgarWords, false) ftr.PushBack(tabula.NewRecordReal(Round(freq))) } }
/* Compute frequency of biased words. */ func (ftr *WordsBiasFrequency) Compute(dataset tabula.DatasetInterface) { col := dataset.GetColumnByName("additions") for _, rec := range col.Records { r := tabula.NewRecordReal(float64(0)) text := rec.String() if len(text) == 0 { ftr.PushBack(r) continue } in := clean.WikiText(text) freq := tekstus.StringFrequenciesOf(in, tekstus.BiasedWords, false) r.SetFloat(freq) ftr.PushBack(r) } }