// we want to extract the 'image/png' from // "data:image/png;base64,..." //func MimeFromBase64(b *bytes.Buffer)(mime string){ func MimeFromBase64(b io.Reader) (mime string) { lg, b := loghttp.BuffLoggerUniversal(nil, nil) _ = b // to avoid huge string allocation // we read the first 100 bytes b1 := make([]byte, 100) _, err := b.Read(b1) lg(err) s := string(b1) pos := base64HeaderPosition(s) if pos > 0 { tmp1 := s[:pos] tmp2 := strings.Split(tmp1, ";") tmp3 := strings.Split(tmp2[0], ":") mime = tmp3[1] } return }
func rssDoc2DirTree(w http.ResponseWriter, r *http.Request, treeX *DirTree, rssDoc RSS, domain string) { lg, lge := loghttp.Logger(w, r) _ = lg if treeX == nil { treeX = &DirTree{Name: "root1", Dirs: map[string]DirTree{}, LastFound: time.Now().Truncate(time.Minute)} } articleList := []FullArticle{} for _, lpItem := range rssDoc.Items.ItemList { t, err := time.Parse(time.RFC1123Z, lpItem.Published) // := time.Parse("Mon, 2 Jan 2006 15:04:05 -0700", lpItem.Published) lge(err) articleList = append(articleList, FullArticle{Url: lpItem.Link, Mod: t}) } lg1, _ := loghttp.BuffLoggerUniversal(w, r) path2DirTree(lg1, treeX, articleList, domain, true) }
func GetHomeTpl(w http.ResponseWriter, r *http.Request, title, body string) *template.Template { if body == "" { body = IDCardHTML + UserInfoHTML } lg, _ := loghttp.BuffLoggerUniversal(w, r) bstpl := tplx.TemplateFromHugoPage(w, r) b := new(bytes.Buffer) fmt.Fprintf(b, tplx.ExecTplHelper(bstpl, map[string]interface{}{ // "HtmlTitle": "{{ .HtmlTitle }}", // this seems to cause problems sometimes. "HtmlTitle": title, "HtmlDescription": "", // reminder "HtmlHeaders": template.HTML(Headers), "HtmlContent": template.HTML(body), })) intHomeTemplate, err := template.New("home").Parse(b.String()) lg(err) return intHomeTemplate }
func queuePush(w http.ResponseWriter, r *http.Request, mx map[string]interface{}) { lg, b := loghttp.BuffLoggerUniversal(w, r) _ = b c := appengine.NewContext(r) m := map[string][]string{"counter_name": []string{nscStringKey}} t := taskqueue.NewPOSTTask("/_ah/namespaced-counters/queue-pop", m) taskqueue.Add(c, t, "") c, err := appengine.Namespace(c, altNamespace) lg(err) taskqueue.Add(c, t, "") io.WriteString(w, "tasks enqueued\n") io.WriteString(w, "\ncounter values now: \n") readBothNamespaces(w, r, mx) io.WriteString(w, "\n\n...sleeping... \n") time.Sleep(time.Duration(400) * time.Millisecond) readBothNamespaces(w, r, mx) }
// unused // probably efficient enough just to call // var bEnc []byte = []byte(sEnc) func StringToVByte(s string) (*bytes.Buffer, *bytes.Buffer) { lg, b := loghttp.BuffLoggerUniversal(nil, nil) _ = b bMsg := new(bytes.Buffer) bDst := new(bytes.Buffer) const chunksize = 20 lb := make([]byte, chunksize) // loop buffer rdr := strings.NewReader(s) for { n1, err := rdr.Read(lb) if err == io.EOF { break } lg(err) if n1 < 1 { break } independentCopy := make([]byte, n1) copy(independentCopy, lb) n2, err := bDst.Write(independentCopy) lg(err) bMsg.WriteString(fmt.Sprintln("reading", n1, "bytes - writing", n2, "bytes: \n")) bMsg.WriteString(fmt.Sprint(" --", string(independentCopy), "--<br>\n")) } return bDst, bMsg }
func readBothNamespaces(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { lg, b := loghttp.BuffLoggerUniversal(w, r) _ = b var c1, c2 int64 var s1, s2 string var err error var reset bool = false p := r.FormValue("reset") if p != "" { reset = true } c := appengine.NewContext(r) c1, err = agnosticReadReset(c, reset) lg(err) { c, err = appengine.Namespace(c, altNamespace) lg(err) c2, err = agnosticReadReset(c, reset) lg(err) } s1 = fmt.Sprintf("%v", c1) s2 = fmt.Sprintf("%v", c2) io.WriteString(w, "|"+s1+"| |"+s2+"|") if reset { io.WriteString(w, " and reset") } }
func searchPut(w http.ResponseWriter, r *http.Request) { lg, b := loghttp.BuffLoggerUniversal(w, r) _ = b id := "PA6-5001" user := &User{ Customer: "Carl Corral", Comment: "I am <em>riled up</em> text", Visits: 1, LastVisit: time.Now(), Birthday: time.Date(1968, time.May, 19, 0, 0, 0, 0, time.UTC), } c := appengine.NewContext(r) index, err := search.Open("users") lg(err) ret_id, err := index.Put(c, id, user) lg(err) fmt.Fprint(w, "OK, saved "+ret_id+"\n\n") var u2 User err = index.Get(c, ret_id, &u2) lg(err) fmt.Fprint(w, "Retrieved document: ", u2) }
/* https://tec-news.appspot.com/coinbase-integr/redir-success1? order[button][description]=When and how Bitcoin decline will start.& order[button][id]=0025d69ea925b48ba2b7adeb2a911ca2& order[button][name]=Bitcoin Analysis& order[button][repeat]=& order[button][resource_path]=/v2/checkouts/4f1e5ecc-c8fc-56fc-926c-15a7eebd8314& order[button][subscription]=false& order[button][type]=buy_now& order[button][uuid]=4f1e5ecc-c8fc-56fc-926c-15a7eebd8314& order[created_at]=2015-10-26 08:03:17 -0700& order[custom]=productID=/member/tec-news/crypto-experts-neglect-one-vital-aspect&uID=14952300052240127534& order[event]=& order[id]=GAB5VN36& order[metadata]=& order[receive_address]=myL84ofiymQpzzmJ7Foc9F2wQ4GMuSuQ3f& order[refund_address]=mwaz3wxMbnZrBZUSZpVHr51xjQ6Swx756b& order[resource_path]=/v2/orders/9bbf6fde-530a-53a4-bf94-d54fc3f43d40& order[status]=completed& order[total_btc][cents]=5600.0& order[total_btc][currency_iso]=BTC& order[total_native][cents]=50.0& order[total_native][currency_iso]=EUR& order[total_payout][cents]=0.0& order[total_payout][currency_iso]=USD& order[transaction][confirmations]=0& order[transaction][hash]=ada26d75ff1e16b4febf539433d5260441171560c57adfff2ac968be37108112& order[transaction][id]=562e40dede472f26be000018& order[uuid]=9bbf6fde-530a-53a4-bf94-d54fc3f43d40 */ func paymentSuccess(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { r.Header.Set("X-Custom-Header-Counter", "nocounter") lg, _ := loghttp.BuffLoggerUniversal(w, r) err := r.ParseForm() if err != nil { lg(err) http.Error(w, err.Error(), http.StatusInternalServerError) return } custom := r.Form.Get("order[custom]") // w.Write([]byte("custom=" + custom + "<br>\n")) values, err := url.ParseQuery(custom) if err != nil { lg(err) http.Error(w, err.Error(), http.StatusInternalServerError) return } productID := values.Get("productID") uID := values.Get("uID") if productID != "" { lg("about to redirect to %v", productID) http.Redirect(w, r, productID+"?redirected-from=paymentsucc", http.StatusFound) return } w.Write([]byte("productID=" + productID + " uID=" + uID + "<br>\n")) }
// Dom2File writes DOM to file func Dom2File(fn string, node *html.Node) { lg, _ := loghttp.BuffLoggerUniversal(nil, nil) var b bytes.Buffer err := html.Render(&b, node) lg(err) Bytes2File(fn, b.Bytes()) }
func queuePop(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { lg, b := loghttp.BuffLoggerUniversal(w, r) _ = b c := appengine.NewContext(r) err := agnosticIncrement(c) c.Infof("qp") lg(err) }
func Base64_str_to_img(base64_img string) (img image.Image, format string) { lg, b := loghttp.BuffLoggerUniversal(nil, nil) _ = b pos := base64HeaderPosition(base64_img) reader := base64.NewDecoder(base64.StdEncoding, strings.NewReader(base64_img[pos:])) img, format, err := image.Decode(reader) lg(err) return }
func deleteAll(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { lg, _ := loghttp.BuffLoggerUniversal(w, r) err := r.ParseForm() lg(err) wpf(w, tplx.ExecTplHelper(tplx.Head, map[string]interface{}{"HtmlTitle": "Delete all filesystem data"})) defer wpf(w, tplx.Foot) confirm := r.FormValue("confirm") if confirm != "yes" { wpf(w, "All dsfs contents are deletes. All memfs contents are deleted<br>\n") wpf(w, "Put a get param into the URL ?confirm - and set it to 'yes'<br>\n") wpf(w, "Put a get param 'mountname' into url; i.e. mountname=mntftch<br>\n") return } wpf(w, "<pre>\n") defer wpf(w, "\n</pre>") // // fs := dsfs.New( dsfs.AeContext(appengine.NewContext(r)), ) mountName := r.FormValue("mountname") if mountName != "" { wpf(w, "mountame = "+mountName+"\n") fs = dsfs.New( dsfs.AeContext(appengine.NewContext(r)), dsfs.MountName(mountName), ) } wpf(w, "dsfs:\n") msg, err := fs.DeleteAll() if err != nil { wpf(w, "err during delete %v\n", err) } wpf(w, msg) memMapFileSys = memfs.New() wpf(w, "\n") wpf(w, "memMapFs new") // cleanup must be manual osFileSys = osfs.New() }
// Bytes2File writes bytes; creates path if neccessary // and logs any errors even to appengine log func Bytes2File(fn string, b []byte) { lg, _ := loghttp.BuffLoggerUniversal(nil, nil) var err error err = ioutil.WriteFile(fn, b, 0) if err != nil { err = os.MkdirAll(filepath.Dir(fn), os.ModePerm) lg(err, "directory creation failed: %v") err = ioutil.WriteFile(fn, b, 0) lg(err) } }
// based on bytes.Buffer and Writing into it func VVByte_to_string(m [][]byte) (*bytes.Buffer, *bytes.Buffer) { lg, b := loghttp.BuffLoggerUniversal(nil, nil) _ = b bRet := new(bytes.Buffer) bMsg := new(bytes.Buffer) //for i,v := range m { for i := 0; i < len(m); i++ { n, err := bRet.Write(m[i]) lg(err) bMsg.WriteString(" lp" + util.Itos(i) + ": writing " + util.Itos(n) + " bytes: \n") } return bRet, bMsg }
func splitPort(hp string) string { lg, b := loghttp.BuffLoggerUniversal(nil, nil) _ = b host, port, err := net.SplitHostPort(hp) _ = port if err != nil { if strings.Contains(err.Error(), "missing port in address") { // normal } else { lg(err) } host = hp } return host }
func String_to_VVByte(base64_img string) ([][]byte, *bytes.Buffer) { lg, _ := loghttp.BuffLoggerUniversal(nil, nil) bMsg := new(bytes.Buffer) const chunksize = 400 // var size_o int if len(base64_img)%chunksize == 0 { size_o = len(base64_img) / chunksize } else { size_o = len(base64_img)/chunksize + 1 } VVByte := make([][]byte, size_o) cntr := -1 b := make([]byte, chunksize) rdr := strings.NewReader(base64_img) for { cntr++ n, err := rdr.Read(b) if err == io.EOF { break } lg(err) if n < 1 { break } indep_copy := make([]byte, n) copy(indep_copy, b) VVByte[cntr] = indep_copy bMsg.WriteString("reading " + util.Itos(n) + " bytes:\n") //bMsg.Write( VVByte[util.Itos(cntr)] ) } return VVByte, bMsg }
func incrementBothNamespaces(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { lg, b := loghttp.BuffLoggerUniversal(w, r) _ = b c := appengine.NewContext(r) err := agnosticIncrement(c) lg(err) { c, err := appengine.Namespace(c, altNamespace) lg(err) err = agnosticIncrement(c) lg(err) } s := `counters updates für ns='' and ns='ns01'.` + "\n" io.WriteString(w, s) readBothNamespaces(w, r, m) }
func TemplateFromHugoPage(w http.ResponseWriter, r *http.Request) string { mp.Lock() if tpl, ok := mp.mp[pathToTmpl]; ok { mp.Unlock() return tpl } mp.Unlock() lg, _ := loghttp.BuffLoggerUniversal(w, r) // fs2 := dsfs.New( dsfs.MountName(TplPrefix[1:]), dsfs.AeContext(appengine.NewContext(r)), ) fs1.SetOption( memfs.ShadowFS(fs2), ) bts, err := fs1.ReadFile(pathToTmpl) if err != nil { lg(err) bts = hugoTplFallback } bts = bytes.Replace(bts, []byte("[REPLACE_TITLE]"), []byte("{{ .HtmlTitle }}"), -1) bts = bytes.Replace(bts, []byte("[REPLACE_DESC]"), []byte("{{ .HtmlDescription }}"), -1) bts = bytes.Replace(bts, []byte("</head>"), []byte("{{ .HtmlHeaders }}\n</head>"), -1) bts = bytes.Replace(bts, []byte("<p>[REPLACE_CONTENT]</p>"), []byte("{{ .HtmlContent }}"), -1) bts = bytes.Replace(bts, []byte("[REPLACE_CONTENT]"), []byte("{{ .HtmlContent }}"), -1) bts = bytes.Replace(bts, []byte("<span id='REPLACE_FOOTER'></span>"), []byte("{{ .HtmlFooter }}"), -1) mp.Lock() mp.mp[pathToTmpl] = string(bts) mp.Unlock() return mp.mp[pathToTmpl] }
func Test2(t *testing.T) { start := time.Now() lg, b := loghttp.BuffLoggerUniversal(nil, nil) _ = b // closureOverBuf := func(bUnused *bytes.Buffer) { // loghttp.Pf(nil, nil, b.String()) // } // defer closureOverBuf(b) // the argument is ignored, var c aetest.Context if false { var err error c, err = aetest.NewContext(nil) lg(err) if err != nil { return } defer c.Close() } fs := GetFS(c, 2) lg("took1 %4.2v secs", time.Now().Sub(start).Seconds()) least3Files := FetchAndDecodeJSON(nil, URLs[0], "", lg, fs) lg("took2 %4.2v secs", time.Now().Sub(start).Seconds()) doc := Dedup(least3Files, lg, fs) fNamer := domclean2.FileNamer(logDir, 0) fNamer() // first call yields key fsPerm := GetFS(c, 2) fileDump(lg, fsPerm, doc, fNamer, "_fin.html") pf("MapSimiliarCompares: %v SimpleCompares: %v LevenstheinComp: %v\n", breakMapsTooDistinct, appliedLevenshtein, appliedCompare) pf("Finish\n") }
// convert image to string, prepend mime header // inspiration_1 http://stackoverflow.com/questions/22945486/golang-converting-image-image-to-byte // inspiration_2 https://github.com/polds/imgbase64/blob/master/images.go // // mime type is always image/png // because we only accept *image.RGBA and use image/png Encoder func Rgba_img_to_base64_str(img *image.RGBA) string { lg, b := loghttp.BuffLoggerUniversal(nil, nil) _ = b // img to bytes buf := new(bytes.Buffer) err := png.Encode(buf, img) lg(err) imgBytes := buf.Bytes() // binary bytes to base64 bytes e64 := base64.StdEncoding maxEncLen := e64.EncodedLen(len(imgBytes)) imgEnc := make([]byte, maxEncLen) e64.Encode(imgEnc, imgBytes) // base64 bytes to string mimeType := "image/png" return fmt.Sprintf("data:%s;base64,%s", mimeType, imgEnc) }
func getHomeTpl(w http.ResponseWriter, r *http.Request) *template.Template { lg, _ := loghttp.BuffLoggerUniversal(w, r) bstpl := tplx.TemplateFromHugoPage(w, r) b := new(bytes.Buffer) fmt.Fprintf(b, tplx.ExecTplHelper(bstpl, map[string]interface{}{ // "HtmlTitle": "{{ .HtmlTitle }}", // this seems to cause problems sometimes. "HtmlTitle": "Member Area", "HtmlDescription": "", // reminder "HtmlHeaders": template.HTML(Headers), "HtmlContent": template.HTML(home1 + "\n<br><br>\n" + home2), })) intHomeTemplate, err := template.New("home").Parse(b.String()) lg(err) return intHomeTemplate }
func GetWidgetTpl(w http.ResponseWriter, r *http.Request, title string) *template.Template { lg, _ := loghttp.BuffLoggerUniversal(w, r) bstpl := tplx.TemplateFromHugoPage(w, r) // the jQuery irritates // bstpl := tplx.HugoTplNoScript b := new(bytes.Buffer) fmt.Fprintf(b, tplx.ExecTplHelper(bstpl, map[string]interface{}{ // "HtmlTitle": "{{ .HtmlTitle }}", // it DOES cause some eternal loop. But why only here? "HtmlTitle": title, "HtmlDescription": "", // reminder "HtmlHeaders": template.HTML(Headers), "HtmlContent": template.HTML(widgetHTML), })) intGitkitTemplate, err := template.New("widg").Parse(b.String()) lg(err) return intGitkitTemplate }
func searchRetrieve(w http.ResponseWriter, r *http.Request) { lg, b := loghttp.BuffLoggerUniversal(w, r) _ = b c := appengine.NewContext(r) index, err := search.Open("users") lg(err) for t := index.Search(c, "Comment:riled", nil); ; { var res User id, err := t.Next(&res) fmt.Fprintf(w, "\n-- ") if err == search.Done { break } if err != nil { fmt.Fprintf(w, "Search error: %v\n", err) break } fmt.Fprintf(w, "%s -> %#v\n", id, res) } }
func confirmPay(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { /* http://abc.de/ef?input_transaction_hash=46178baf7de078954b5aebb71c12120b33d998faac1c165af195eae90f19b25c&shared=false&address=18tpXf8WWuhJP95JbDASbZvavmZJbrydut&destination_address=18tpXf8WWuhJP95JbDASbZvavmZJbrydut&input_address=1ZTnjSdknZvur9Gc73gvB8XBTWL7nV1m6&test=true&anonymous=false&confirmations=0&value=82493362&transaction_hash=46178baf7de078954b5aebb71c12120b33d998faac1c165af195eae90f19b25c */ lg, b := loghttp.BuffLoggerUniversal(w, r) closureOverBuf := func(bUnused *bytes.Buffer) { loghttp.Pf(w, r, b.String()) } defer closureOverBuf(b) // the argument is ignored, r.Header.Set("X-Custom-Header-Counter", "nocounter") wpf(b, tplx.ExecTplHelper(tplx.Head, map[string]interface{}{"HtmlTitle": "Payment confirmation"})) defer wpf(b, tplx.Foot) wpf(b, "<pre>") defer wpf(b, "</pre>") err := r.ParseForm() lg(err) custSecret := "" if r.FormValue("customsecret") != "" { custSecret = r.FormValue("customsecret") } lg("custom secret is %q", custSecret) val := "" if r.FormValue("value") != "" { val = r.FormValue("value") } lg("value is %q", val) }
// // https://developers.google.com/identity/choose-auth // https://developers.google.com/identity/sign-in/web/backend-auth func TokenSignin(w http.ResponseWriter, r *http.Request) { lg, _ := loghttp.BuffLoggerUniversal(w, r) // w.Header().Set("Access-Control-Allow-Origin", "http://localhost:1313") w.Header().Set("Access-Control-Allow-Origin", "http://"+routes.AppHostDev()) w.Header().Del("Access-Control-Allow-Origin") w.Header().Set("Access-Control-Allow-Origin", "*") // err := r.ParseMultipartForm(1024 * 1024 * 2) err := r.ParseForm() lg(err) myToken := r.Form.Get("idtoken") tokSize := fmt.Sprintf("Len of Tok was %v. \n", len(myToken)) fc1 := func(token *jwt.Token) (interface{}, error) { // Don't forget to validate the alg is what you expect: log.Printf("algo header is %v\n", token.Header["alg"]) if _, ok := token.Method.(*jwt.SigningMethodRSA); !ok { return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"]) } return token.Header["kid"], nil } token, err := jwt.Parse(myToken, fc1) // No direct error comparison possible; since err is wrapped in another struct if err != nil && strings.Contains(err.Error(), jwt.ErrPEMMappingObsolete.Error()) { currentPEMsURL := "https://www.googleapis.com/oauth2/v1/certs" req, err := http.NewRequest("GET", currentPEMsURL, nil) if err != nil { lg("creation of pem request failed") return } req.Header.Set("Content-Type", "application/json") fo := fetch.Options{Req: req} fo.KnownProtocol = "https" fo.ForceHTTPSEvenOnDevelopmentServer = true bts, inf, err := fetch.UrlGetter(r, fo) lg(err) if err != nil { lg("tried to fetch %v, %v", currentPEMsURL, inf.URL) lg("msg %v", inf.Msg) return } if len(bts) > 200 { var data1 map[string]string err = json.Unmarshal(bts, &data1) lg(err) // lg(stringspb.IndentedDumpBytes(data1)) // w.Write(stringspb.IndentedDumpBytes(data1)) if len(data1) > 1 { lg("PEM mappings updated") jwt.MappingToPEM = data1 } else { lg("PEM mapping response contained only %v records; bytes length %v", len(data1), len(bts)) } } } token, err = jwt.Parse(myToken, fc1) if err != nil && strings.Contains(err.Error(), jwt.ErrInvalidKey.Error()) { w.Write([]byte("The submitted RSA Key was somehow unparseable. We still accept the token.\n")) /* https://developers.google.com/identity/sign-in/web/backend-auth */ err = nil token.Valid = true } if err != nil { w.Write([]byte("--- " + err.Error() + ".\n")) } if err == nil && token.Valid { tk := "" tk += fmt.Sprintf(" Algor: %v\n", token.Method) tk += fmt.Sprintf(" Header: %v\n", token.Header) for k, v := range token.Claims { tk += fmt.Sprintf("\t %-8v %v\n", k, v) } lg(tk) w.Write([]byte("tokensignin; valid. \n")) w.Write([]byte(tokSize)) sb := "header-sub-not-present" if _, ok := token.Claims["sub"]; ok { sb = token.Claims["sub"].(string) } w.Write([]byte("ID from PWT is " + sb + "\n")) _, usr, msg1 := login.CheckForNormalUser(r) if usr != nil { w.Write([]byte("ID from SRV is " + usr.ID + "\n")) } w.Write([]byte(msg1 + "\n")) } else { w.Write([]byte("tokensignin; INVALID. \n")) w.Write([]byte(tokSize)) w.Write([]byte(stringspb.ToLen(myToken, 30))) vrf := fmt.Sprintf("\nhttps://www.googleapis.com/oauth2/v3/tokeninfo?id_token=%v \n", myToken) w.Write([]byte(vrf)) } }
func fetchSimForm(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { lg, b := loghttp.BuffLoggerUniversal(w, r) closureOverBuf := func(bUnused *bytes.Buffer) { loghttp.Pf(w, r, b.String()) } defer closureOverBuf(b) // the argument is ignored, r.Header.Set("X-Custom-Header-Counter", "nocounter") // on live server => always use https if r.URL.Scheme != "https" && !util_appengine.IsLocalEnviron() { r.URL.Scheme = "https" r.URL.Host = r.Host lg("lo - redirect %v", r.URL.String()) http.Redirect(w, r, r.URL.String(), http.StatusFound) } err := r.ParseForm() lg(err) rURL := "" if r.FormValue(routes.URLParamKey) != "" { rURL = r.FormValue(routes.URLParamKey) } if len(rURL) == 0 { wpf(b, tplx.ExecTplHelper(tplx.Head, map[string]interface{}{"HtmlTitle": "Find similar HTML URLs"})) defer wpf(b, tplx.Foot) tm := map[string]string{ "val": "www.welt.de/politik/ausland/article146154432/Tuerkische-Bodentruppen-marschieren-im-Nordirak-ein.html", "fieldname": routes.URLParamKey, } tplForm := tt.Must(tt.New("tplName01").Parse(htmlForm)) tplForm.Execute(b, tm) } else { fullURL := fmt.Sprintf("https://%s%s?%s=%s&cnt=%s&prot=%s", r.Host, routes.FetchSimilarURI, routes.URLParamKey, rURL, r.FormValue("cnt"), r.FormValue("prot")) lg("lo - sending to URL 1: %v", fullURL) fo := fetch.Options{} fo.URL = fullURL bts, inf, err := fetch.UrlGetter(r, fo) _ = inf lg(err) if err != nil { return } if len(bts) == 0 { lg("empty bts") return } var mp map[string][]byte err = json.Unmarshal(bts, &mp) lg(err) if err != nil { lg("%s", bts) return } w.Header().Set("Content-Type", "text/html; charset=utf-8") if _, ok := mp["msg"]; ok { w.Write(mp["msg"]) } for k, v := range mp { if k != "msg" { wpf(w, "<br><br>%s:\n", k) if true { wpf(w, "len %v", len(v)) } else { wpf(w, "%s", html.EscapeString(string(v))) } } } } }
func requestPay(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { lg, b := loghttp.BuffLoggerUniversal(w, r) closureOverBuf := func(bUnused *bytes.Buffer) { loghttp.Pf(w, r, b.String()) } defer closureOverBuf(b) // the argument is ignored, r.Header.Set("X-Custom-Header-Counter", "nocounter") protoc := "https://" if appengine.IsDevAppServer() { protoc = "http://" } host := appengine.DefaultVersionHostname(appengine.NewContext(r)) if appengine.IsDevAppServer() { host = "not-loclhost" } confirmURL := fmt.Sprintf("%v%v%v", protoc, host, uriConfirmPayment) confirmURL = url.QueryEscape(confirmURL) addrURL := fmt.Sprintf("https://%v/api/receive?method=create&address=%v&callback=%v&customsecret=49&api_code=%v", blockChainHost, bitCoinAddress, confirmURL, apiKey) req, err := http.NewRequest("GET", addrURL, nil) lg(err) if err != nil { return } bts, inf, err := fetch.UrlGetter(r, fetch.Options{Req: req}) bts = bytes.Replace(bts, []byte(`","`), []byte(`", "`), -1) if err != nil { lg(err) lg(inf.Msg) return } lg("response body 1:\n") lg("%s\n", string(bts)) lg("response body 2:\n") var data1 map[string]interface{} err = json.Unmarshal(bts, &data1) lg(err) lg(stringspb.IndentedDumpBytes(data1)) // lg("%#v", data1) inputAddress, ok := data1["input_address"].(string) if !ok { lg("input address could not be casted to string; is type %T", data1["input_address"]) return } feePercent, ok := data1["fee_percent"].(float64) if !ok { lg("fee percent could not be casted to float64; is type %T", data1["fee_percent"]) return } lg("Input Adress will be %q; fee percent will be %4.2v", inputAddress, feePercent) }
// FetchSimilar is an extended version of Fetch // It is uses a DirTree of crawled *links*, not actual files. // As it moves up the DOM, it crawls every document for additional links. // It first moves up to find similar URLs on the same depth // /\ // /\ / \ // /\ / \ / \ // It then moves up the ladder again - to accept higher URLs // /\ // /\ // /\ func FetchSimilar(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { lg, b := loghttp.BuffLoggerUniversal(w, r) closureOverBuf := func(bUnused *bytes.Buffer) { loghttp.Pf(w, r, b.String()) } defer closureOverBuf(b) // the argument is ignored, r.Header.Set("X-Custom-Header-Counter", "nocounter") start := time.Now() wpf(b, tplx.ExecTplHelper(tplx.Head, map[string]interface{}{"HtmlTitle": "Find similar HTML URLs"})) defer wpf(b, tplx.Foot) wpf(b, "<pre>") defer wpf(b, "</pre>") fs1 := GetFS(appengine.NewContext(r)) err := r.ParseForm() lg(err) countSimilar := 3 sCountSimilar := r.FormValue("cnt") if sCountSimilar != "" { i, err := strconv.Atoi(strings.TrimSpace(sCountSimilar)) if err == nil { countSimilar = i } } surl := r.FormValue(routes.URLParamKey) ourl, err := fetch.URLFromString(surl) lg(err) if err != nil { return } if ourl.Host == "" { lg("host is empty (%v)", surl) return } knownProtocol := "" if r.FormValue("prot") != "" { knownProtocol = r.FormValue("prot") } numWorkers := 0 sNumWorkers := r.FormValue("numworkers") if sNumWorkers != "" { i, err := strconv.Atoi(strings.TrimSpace(sNumWorkers)) if err == nil { numWorkers = i } } srcDepth := strings.Count(ourl.Path, "/") cmd := FetchCommand{} cmd.Host = ourl.Host cmd.SearchPrefix = ourl.Path cmd = addDefaults(cmd) dirTree := &DirTree{Name: "/", Dirs: map[string]DirTree{}, EndPoint: true} fnDigest := path.Join(docRoot, cmd.Host, "digest2.json") loadDigest(w, r, lg, fs1, fnDigest, dirTree) // previous lg("dirtree 400 chars is %v end of dirtree\t\t", stringspb.ToLen(dirTree.String(), 400)) m1 := new(MyWorker) m1.r = r m1.lg = lg m1.fs1 = fs1 m1.SURL = path.Join(cmd.Host, ourl.Path) m1.Protocol = knownProtocol btsSrc, modSrc, usedExisting, err := fetchSave(m1) if !usedExisting { addAnchors(lg, cmd.Host, btsSrc, dirTree) } lg(err) if err != nil { return } lg("\t\t%4.2v secs so far 1", time.Now().Sub(start).Seconds()) var treePath string treePath = "/blogs/freeexchange" treePath = "/news/europe" treePath = path.Dir(ourl.Path) opt := LevelWiseDeeperOptions{} opt.Rump = treePath opt.ExcludeDir = "/news/americas" opt.ExcludeDir = "/blogs/buttonwood" opt.ExcludeDir = "/something-impossible" opt.MinDepthDiff = 1 opt.MaxDepthDiff = 1 opt.CondenseTrailingDirs = cmd.CondenseTrailingDirs opt.MaxNumber = cmd.DesiredNumber + 1 // one more for "self" opt.MaxNumber = cmd.DesiredNumber + 40 // collect more, 'cause we filter out those too old later var subtree *DirTree links := []FullArticle{} alreadyCrawled := map[string]struct{}{} MarkOuter: for j := 0; j < srcDepth; j++ { treePath = path.Dir(ourl.Path) MarkInner: // for i := 1; i < srcDepth; i++ { for i := 1; i < (srcDepth + 5); i++ { subtree, treePath = DiveToDeepestMatch(dirTree, treePath) lg("Looking from height %v to level %v - %v", srcDepth-i, srcDepth-j, treePath) if _, ok := alreadyCrawled[treePath]; ok { // lg("\t already digested %v", treePath) continue } m2 := new(MyWorker) m2.r = r m2.lg = lg m2.fs1 = fs1 m2.SURL = path.Join(cmd.Host, treePath) m2.Protocol = knownProtocol btsPar, _, usedExisting, err := fetchSave(m2) lg(err) if err != nil { return } alreadyCrawled[treePath] = struct{}{} if !usedExisting { addAnchors(lg, cmd.Host, btsPar, dirTree) } if subtree == nil { lg("\n#%v treePath %q ; subtree is nil", i, treePath) } else { // lg("\n#%v treePath %q ; subtree exists", i, treePath) opt.Rump = treePath opt.MinDepthDiff = i - j opt.MaxDepthDiff = i - j lvlLinks := LevelWiseDeeper(nil, nil, subtree, opt) links = append(links, lvlLinks...) for _, art := range lvlLinks { _ = art // lg("#%v fnd %v", i, stringspb.ToLen(art.Url, 100)) } if len(links) >= opt.MaxNumber { lg("found enough links") break MarkOuter } pathPrev := treePath treePath = path.Dir(treePath) // lg("#%v bef %v - aft %v", i, pathPrev, treePath) if pathPrev == "." && treePath == "." || pathPrev == "/" && treePath == "/" || pathPrev == "" && treePath == "." { lg("break to innner") break MarkInner } } } } // // // // lg("%v links after %4.2v secs", len(links), time.Now().Sub(start).Seconds()) lg("============================") lg("Now reading/fetching actual similar files - not just the links") // tried := 0 selecteds := []FullArticle{} nonExisting := []FullArticle{} nonExistFetched := []FullArticle{} for _, art := range links { if art.Url == ourl.Path { lg("skipping self\t%v", art.Url) continue } tried++ useExisting := false semanticUri := condenseTrailingDir(art.Url, cmd.CondenseTrailingDirs) p := path.Join(docRoot, cmd.Host, semanticUri) f, err := fs1.Open(p) // lg(err) // its no error if file does not exist if err != nil { // lg("!nstore %q", semanticUri) } else { // lg("reading %q", semanticUri) // lets put this into a func, so that f.close it called at the end of this func // otherwise defer f.close() spans the entire func and prevents // overwrites chmods further down f := func() { defer f.Close() fi, err := f.Stat() lg(err) if err != nil { } else { age := time.Now().Sub(fi.ModTime()) if age.Hours() < 10 { lg("\t\tusing existing file with age %4.2v hrs", age.Hours()) art.Mod = fi.ModTime() bts, err := ioutil.ReadAll(f) lg(err) art.Body = bts if len(bts) < 200 { if bytes.Contains(bts, []byte(fetch.MsgNoRdirects)) { return } } selecteds = append(selecteds, art) useExisting = true } } } f() } if !useExisting { nonExisting = append(nonExisting, art) } if len(selecteds) >= countSimilar { break } } lg("============================") lg("tried %v links - yielding %v existing similars; not existing in datastore: %v, %v were requested.", tried, len(selecteds), len(nonExisting), countSimilar) if len(selecteds) < countSimilar { jobs := make([]distrib.Worker, 0, len(nonExisting)) for _, art := range nonExisting { surl := path.Join(cmd.Host, art.Url) wrkr := MyWorker{SURL: surl} wrkr.Protocol = knownProtocol wrkr.r = r wrkr.lg = lg wrkr.fs1 = fs1 job := distrib.Worker(&wrkr) jobs = append(jobs, job) } opt := distrib.NewDefaultOptions() opt.TimeOutDur = 3500 * time.Millisecond opt.Want = int32(countSimilar - len(selecteds) + 4) // get some more, in case we have "redirected" bodies opt.NumWorkers = int(opt.Want) // 5s query limit; => hurry; spawn as many as we want if numWorkers > 0 { opt.NumWorkers = numWorkers } lg("Preparing %v simultaneous, wanting %v fetches; at %4.2v secs.", opt.NumWorkers, opt.Want, time.Now().Sub(start).Seconds()) opt.CollectRemainder = false // 5s query limit; => hurry; dont wait for stragglers ret, msg := distrib.Distrib(jobs, opt) lg("Distrib returned at %4.2v secs with %v results.", time.Now().Sub(start).Seconds(), len(ret)) lg("\n" + msg.String()) for _, v := range ret { v1, _ := v.Worker.(*MyWorker) if v1.FA != nil { age := time.Now().Sub(v1.FA.Mod) if age.Hours() < 10 { lg("\t\tusing fetched file with age %4.2v hrs", age.Hours()) nonExistFetched = append(nonExistFetched, *v1.FA) if len(nonExistFetched) > (countSimilar - len(selecteds)) { break } } } if v1.err != nil { lg(err) } } lg("tried %v links - yielding %v fetched - jobs %v", len(nonExisting), len(nonExistFetched), len(jobs)) selecteds = append(selecteds, nonExistFetched...) // // // Extract links for _, v := range nonExistFetched { // lg("links -> memory dirtree for %q", v.Url) addAnchors(lg, cmd.Host, v.Body, dirTree) } } // if time.Now().Sub(dirTree.LastFound).Seconds() < 10 { lg("saving accumulated (new) links to digest") saveDigest(lg, fs1, fnDigest, dirTree) } lg("\t\t%4.2v secs so far 3", time.Now().Sub(start).Seconds()) mp := map[string][]byte{} mp["msg"] = b.Bytes() mp["url_self"] = []byte(condenseTrailingDir(ourl.Path, cmd.CondenseTrailingDirs)) mp["mod_self"] = []byte(modSrc.Format(http.TimeFormat)) mp["bod_self"] = btsSrc for i, v := range selecteds { mp["url__"+spf("%02v", i)] = []byte(v.Url) mp["mod__"+spf("%02v", i)] = []byte(v.Mod.Format(http.TimeFormat)) mp["bod__"+spf("%02v", i)] = v.Body } mp["lensimilar"] = []byte(spf("%02v", len(selecteds))) // smp, err := json.MarshalIndent(mp, "", "\t") if err != nil { lg(b, "marshalling mp to []byte failed\n") return } r.Header.Set("X-Custom-Header-Counter", "nocounter") w.Header().Set("Content-Type", "application/json") w.Write(smp) b.Reset() // this keeps the buf pointer intact; outgoing defers are still heeded b = new(bytes.Buffer) // creates a *new* buf pointer; outgoing defers write into the *old* buf lg("\t\t%4.2v secs so far 4 (json resp written as []byte)", time.Now().Sub(start).Seconds()) return }
// BytesFromFile reads bytes and logs any // errors even to appengine log. func BytesFromFile(fn string) []byte { lg, _ := loghttp.BuffLoggerUniversal(nil, nil) b, err := ioutil.ReadFile(fn) lg(err) return b }
// dedupHTTP wraps Dedup() func dedupHTTP(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { lg, b := loghttp.BuffLoggerUniversal(w, r) closureOverBuf := func(bUnused *bytes.Buffer) { loghttp.Pf(w, r, b.String()) } defer closureOverBuf(b) // the argument is ignored, r.Header.Set("X-Custom-Header-Counter", "nocounter") wpf(b, tplx.ExecTplHelper(tplx.Head, map[string]interface{}{"HtmlTitle": "Deduplicating redundant stuff"})) defer wpf(b, tplx.Foot) wpf(b, "<pre>") defer wpf(b, "</pre>") err := r.ParseForm() lg(err) surl := r.FormValue(routes.URLParamKey) ourl, err := fetch.URLFromString(surl) lg(err) if err != nil { return } if ourl.Host == "" { lg("host is empty (%v)", surl) return } knownProtocol := "" if r.FormValue("prot") != "" { knownProtocol = r.FormValue("prot") } lg("Host %q, Path %q", ourl.Host, ourl.Path) fs := GetFS(appengine.NewContext(r), 0) least3Files := FetchAndDecodeJSON(r, ourl.String(), knownProtocol, lg, fs) lg("Fetched and decoded; found %v", len(least3Files)) if len(least3Files) > 0 { doc := Dedup(ourl, least3Files, lg, fs) fNamer := domclean2.FileNamer(logDir, 0) fNamer() // first call yields key fsPerm := GetFS(appengine.NewContext(r), 0) fileDump(lg, fsPerm, doc, fNamer, "_fin.html") lg("MapSimiliarCompares: %v SimpleCompares: %v LevenstheinComp: %v\n", breakMapsTooDistinct, appliedLevenshtein, appliedCompare) lg("Finish\n") var b2 bytes.Buffer err := html.Render(&b2, doc) lg(err) if err != nil { return } b = new(bytes.Buffer) // w.Write([]byte("aa")) w.Header().Set("Content-type", "text/html; charset=utf-8") w.Write(b2.Bytes()) } }