func (this *NLPEngine) Workflow(document *models.DocumentEntity, output chan *models.DocumentEntity) { defer func() { if r := recover(); r != nil { err, _ := r.(error) if err != nil { output <- nil //err.Error() } else { output <- nil } } }() document.Init() tokens := list.New() url := document.Url content := document.Content if url != "" && content == "" { crawler := NewDefaultCrawler() article := crawler.Analyze(url) document.Title = article.Title document.Description = article.MetaDescription document.Keywords = article.MetaKeywords document.TopImage = article.TopImage document.Content = article.CleanedText } body := StringsAppend(document.Title, document.Description, document.Keywords, document.Content) if this.tokenizer != nil { this.tokenizer.Tokenize(body, 0, tokens) } sentences := list.New() if this.splitter != nil { sid := this.splitter.OpenSession() this.splitter.Split(sid, tokens, true, sentences) this.splitter.CloseSession(sid) } for ss := sentences.Front(); ss != nil; ss = ss.Next() { s := ss.Value.(*Sentence) if this.morfo != nil { this.morfo.Analyze(s) } if this.sense != nil { this.sense.Analyze(s) } if this.tagger != nil { this.tagger.Analyze(s) } if this.shallowParser != nil { this.shallowParser.Analyze(s) } } if this.dsb != nil { this.dsb.Analyze(sentences) } entities := make(map[string]int64) for ss := sentences.Front(); ss != nil; ss = ss.Next() { se := models.NewSentenceEntity() body := "" s := ss.Value.(*Sentence) for ww := s.Front(); ww != nil; ww = ww.Next() { w := ww.Value.(*Word) a := w.Front().Value.(*Analysis) te := models.NewTokenEntity(w.getForm(), a.getLemma(), a.getTag(), a.getProb()) if a.getTag() == TAG_NP { entities[w.getForm()]++ } body += w.getForm() + " " se.AddTokenEntity(te) } body = strings.Trim(body, " ") se.SetBody(body) se.SetSentence(s) document.AddSentenceEntity(se) } tempEntities := set.New() mitieEntities := this.mitie.Process(body) for e := mitieEntities.Front(); e != nil; e = e.Next() { entity := e.Value.(*models.Entity) tempEntities.Add(entity.GetValue()) } for name, frequency := range entities { name = strings.Replace(name, "_", " ", -1) if !tempEntities.Has(name) { document.AddUnknownEntity(name, frequency) } } document.Entities = mitieEntities output <- document }