func Test1(t *testing.T) { lg, b := loghttp.BuffLoggerUniversal(nil, nil) _ = b c, err := aetest.NewContext(nil) lg(err) if err != nil { return } defer c.Close() fs := GetFS(c, 2) remoteHostname := "www.welt.de" remoteHostname = "www.welt.de/politik/ausland" dirs1, _, msg, err := fileserver.GetDirContents(repo.RepoURL, remoteHostname) if err != nil { lg(err) lg("%s", msg) } lg("dirs1") for _, v := range dirs1 { lg(" %v", v) } least3URLs := []string{} for _, v1 := range dirs1 { p := path.Join(remoteHostname, v1) dirs2, fils2, msg, err := fileserver.GetDirContents(repo.RepoURL, p) _ = dirs2 if err != nil { lg(err) lg("%s", msg) } // lg(" dirs2 %v", stringspb.IndentedDump(dirs2)) // lg(" fils2 %v", stringspb.IndentedDump(fils2)) for _, v2 := range fils2 { least3URLs = append(least3URLs, path.Join(remoteHostname, v1, v2)) } } if len(least3URLs) < numTotal { lg("not enough files in rss fetcher cache") return } else { least3URLs = least3URLs[:numTotal+1] } lg("fils2") for _, v := range least3URLs { lg(" %v", v) } // domclean least3Files := make([]repo.FullArticle, 0, len(least3URLs)) for i := 0; i < len(least3URLs); i++ { surl := spf("%v/%v", repo.RepoURL, least3URLs[i]) fNamer := domclean2.FileNamer(logDir, i) fNamer() // first call yields key resBytes, inf, err := fetch.UrlGetter(nil, fetch.Options{URL: surl}) if err != nil { lg(err) return } lg("fetched %4.1fkB from %v", float64(len(resBytes))/1024, stringspb.ToLenR(inf.URL.String(), 60)) fa := repo.FullArticle{} fa.Url = inf.URL.String() fa.Mod = inf.Mod fa.Body = resBytes least3Files = append(least3Files, fa) } doc := Dedup(least3Files, lg, fs) fNamer := domclean2.FileNamer(logDir, 0) fNamer() // first call yields key fsPerm := GetFS(c, 2) fileDump(lg, fsPerm, doc, fNamer, "_fin.html") pf("MapSimiliarCompares: %v SimpleCompares: %v LevenstheinComp: %v\n", breakMapsTooDistinct, appliedLevenshtein, appliedCompare) pf("correct finish\n") }
// Puttting it all together func Dedup(oURL *url.URL, least3Files []repo.FullArticle, lg loghttp.FuncBufUniv, fs fsi.FileSystem) *html.Node { opts := domclean2.CleaningOptions{Proxify: true, Beautify: true} // opts.FNamer = fNamer opts.AddOutline = true // opts.RemoteHost = fetch.HostFromStringUrl(least3Files[0].Url) opts.RemoteHost = oURL.Host // // domclean for i := 0; i < len(least3Files); i++ { fNamer := domclean2.FileNamer(logDir, i) fNamer() // first call yields key lg("cleaning %4.1fkB from %v", float64(len(least3Files[i].Body))/1024, stringspb.ToLenR(least3Files[i].Url, 60)) doc, err := domclean2.DomClean(least3Files[i].Body, opts) lg(err) fileDump(lg, fs, doc, fNamer, ".html") } if false { // // Textify with brute force for i := 0; i < len(least3Files); i++ { fNamer := domclean2.FileNamer(logDir, i) fNamer() // first call yields key bts, err := fs.ReadFile(fNamer() + ".html") lg(err) doc, err := html.Parse(bytes.NewReader(bts)) lg(err) textifyBruteForce(doc) var buf bytes.Buffer err = html.Render(&buf, doc) lg(err) b := buf.Bytes() b = bytes.Replace(b, []byte("[br]"), []byte("\n"), -1) fileDump(lg, fs, b, fNamer, "_raw.txt") } } // // Textify with more finetuning. // Save result to memory. textsByArticOutl := map[string][]*TextifiedTree{} for i := 0; i < len(least3Files); i++ { fNamer := domclean2.FileNamer(logDir, i) fnKey := fNamer() // first call yields key bts, err := fs.ReadFile(fNamer() + ".html") doc, err := html.Parse(bytes.NewReader(bts)) lg(err) fNamer() // one more // mp, bts := BubbledUpTextExtraction(doc, fnKey) fileDump(lg, fs, bts, fNamer, ".txt") mpSorted, dump := orderByOutline(mp) fileDump(lg, fs, dump, fNamer, ".txt") textsByArticOutl[fnKey] = mpSorted // for k, v := range mpSorted { // if k%33 != 0 { // continue // } // log.Printf("%3v: %v %14v %v\n", k, v.SourceID, v.Outline, v.Lvl) // } } // // // We progress from level 1 downwards. // Lower levels skip weeded out higher levels, // to save expensive levenshtein comparisons var skipPrefixes = map[string]bool{} for weedStage := 1; weedStage <= stageMax; weedStage++ { fNamer := domclean2.FileNamer(logDir, 0) fnKey := fNamer() // first call yields key levelsToProcess = map[int]bool{weedStage: true} frags := similarTextifiedTrees(textsByArticOutl, skipPrefixes, map[string]bool{fnKey: true}) similaritiesToFile(fs, logDir, frags, weedStage) for _, frag := range frags { if len(frag.Similars) >= numTotal-1 && frag.SumRelLevenshtein/(numTotal-1) < 0.2 { skipPrefixes[frag.Outline+"."] = true } } b := new(bytes.Buffer) for k, _ := range skipPrefixes { b.WriteString(k) b.WriteByte(32) } // log.Printf("%v\n", b.String()) } // // Apply dedup fNamer := domclean2.FileNamer(logDir, 0) fNamer() // first call yields key bts, err := fs.ReadFile(fNamer() + ".html") lg(err) doc, err := html.Parse(bytes.NewReader(bts)) lg(err) dedupApply(doc, skipPrefixes) // A special after dedup cleaning: // Remove ol and cfrm attributes var fr func(*html.Node) fr = func(n *html.Node) { if n.Type == html.ElementNode { attr2 := make([]html.Attribute, 0, len(n.Attr)) for _, attr := range n.Attr { if attr.Key != "ol" && attr.Key != "cfrm" { attr2 = append(attr2, attr) } } n.Attr = attr2 } for c := n.FirstChild; c != nil; c = c.NextSibling { fr(c) } } fr(doc) if false { // does not add value var b7 bytes.Buffer err := html.Render(&b7, doc) lg(err) doc, err = domclean2.DomClean(b7.Bytes(), opts) lg(err) } else { domclean2.DomFormat(doc) } return doc }
func Test1(t *testing.T) { lg, lge := loghttp.Logger(nil, nil) // c := prepare(t) // defer c.Close() lg("waiting for webserver") time.Sleep(2 * time.Millisecond) remoteHostname := "www.welt.de" dirs1, _, msg, err := fileserver.GetDirContents(hostWithPref, remoteHostname) if err != nil { lge(err) lg("%s", msg) } lg("dirs1") for _, v := range dirs1 { lg(" %v", v) } least3Files := []string{} for _, v1 := range dirs1 { dirs2, fils2, msg, err := fileserver.GetDirContents(hostWithPref, path.Join(remoteHostname, v1)) _ = dirs2 if err != nil { lge(err) lg("%s", msg) } // lg(" dirs2 %v", stringspb.IndentedDump(dirs2)) // lg(" fils2 %v", stringspb.IndentedDump(fils2)) if len(fils2) > numTotal-1 { for i2, v2 := range fils2 { least3Files = append(least3Files, path.Join(remoteHostname, v1, v2)) if i2 == numTotal-1 { break } } break } } if len(least3Files) < numTotal { lg("not enough files in rss fetcher cache") return } lg("fils2") for _, v := range least3Files { lg(" %v", v) } logdir := prepareLogDir() iter := make([]int, numTotal) for i, _ := range iter { surl := spf("%v/%v", hostWithPref, least3Files[i]) fNamer := FileNamer(logdir, i) fnKey := fNamer() // first call yields key _ = fnKey resBytes, effUrl, err := fetch.UrlGetter(nil, fetch.Options{URL: surl}) if err != nil { lge(err) return } lg("fetched %4.1fkB from %v", float64(len(resBytes))/1024, stringspb.ToLenR(effUrl.String(), 60)) opts := CleaningOptions{Proxify: true} opts.FNamer = fNamer opts.RemoteHost = remoteHostname doc, err := DomClean(resBytes, opts) lge(err) _ = doc } // statistics on elements and attributes sorted1 := sortmap.SortMapByCount(attrDistinct) sorted1.Print(6) fmt.Println() sorted2 := sortmap.SortMapByCount(nodeDistinct) sorted2.Print(6) pf("correct finish\n") }