func FetchAndDecodeJSON(r *http.Request, surl, knownProtocol string, lg loghttp.FuncBufUniv, fs fsi.FileSystem) []repo.FullArticle { fullURL := fmt.Sprintf("%s%s?%s=%s&cnt=%v&prot=%v", routes.AppHost(), routes.FetchSimilarURI, routes.URLParamKey, surl, numTotal-1, knownProtocol) // fullURL = fmt.Sprintf("%s%s?%s=%s&cnt=%v", r.URL.Host, repo.routes.FetchSimilarURI, // routes.URLParamKey, surl, numTotal-1) lg("lo fetching %v", fullURL) start := time.Now() fo := fetch.Options{} fo.URL = fullURL bJSON, inf, err := fetch.UrlGetter(r, fo) _ = inf lg(err) if err != nil { lg("msg %v", inf.Msg) return nil } if len(bJSON) == 0 { lg("empty bJSON") return nil } lg("\t\tfetch resp complete after %4.2v secs; %vkB", time.Now().Sub(start).Seconds(), len(bJSON)/1024) var mp map[string][]byte err = json.Unmarshal(bJSON, &mp) lg(err) if err != nil { if _, ok := mp["msg"]; ok { lg("%s", mp["msg"]) } else { lg("%s", bJSON) } return nil } smaxFound := string(mp["lensimilar"]) maxFound := util.Stoi(smaxFound) if maxFound < numTotal-1 { lg("not enough files returned by FetchSimilar 1 - mp[lensimilar] too small: %s", mp["lensimilar"]) return nil } least3Files := make([]repo.FullArticle, maxFound+1) _, ok1 := mp["url_self"] _, ok2 := mp["mod_self"] _, ok3 := mp["bod_self"] if ok1 && ok2 && ok3 { least3Files[0].Url = string(mp["url_self"]) least3Files[0].Mod, err = time.Parse(http.TimeFormat, string(mp["mod_self"])) lg(err) least3Files[0].Body = mp["bod_self"] if len(least3Files[0].Body) < 200 { if !bytes.Contains(least3Files[0].Body, []byte(fetch.MsgNoRdirects)) { lg("found base but its a redirect") return nil } } } lg("found base") for k, v := range mp { if k == "msg" { continue } if strings.HasSuffix(k, "self") { continue } if strings.HasPrefix(k, "url__") { sval := strings.TrimPrefix(k, "url__") val := util.Stoi(sval) // lg("%v %v %s", sval, val, v) least3Files[val+1].Url = string(v) } if strings.HasPrefix(k, "mod__") { sval := strings.TrimPrefix(k, "mod__") val := util.Stoi(sval) // lg("%v %v %s", sval, val, v) least3Files[val+1].Mod, err = time.Parse(http.TimeFormat, string(v)) lg(err) } if strings.HasPrefix(k, "bod__") { sval := strings.TrimPrefix(k, "bod__") val := util.Stoi(sval) least3Files[val+1].Body = v //html.EscapeString(string(v) } } lg("found %v similar; decoding complete after %4.2v secs", maxFound, time.Now().Sub(start).Seconds()) for _, v := range least3Files { lg("%v %v", v.Url, len(v.Body)) } return least3Files }
// // https://developers.google.com/identity/choose-auth // https://developers.google.com/identity/sign-in/web/backend-auth func TokenSignin(w http.ResponseWriter, r *http.Request) { lg, _ := loghttp.BuffLoggerUniversal(w, r) // w.Header().Set("Access-Control-Allow-Origin", "http://localhost:1313") w.Header().Set("Access-Control-Allow-Origin", "http://"+routes.AppHostDev()) w.Header().Del("Access-Control-Allow-Origin") w.Header().Set("Access-Control-Allow-Origin", "*") // err := r.ParseMultipartForm(1024 * 1024 * 2) err := r.ParseForm() lg(err) myToken := r.Form.Get("idtoken") tokSize := fmt.Sprintf("Len of Tok was %v. \n", len(myToken)) fc1 := func(token *jwt.Token) (interface{}, error) { // Don't forget to validate the alg is what you expect: log.Printf("algo header is %v\n", token.Header["alg"]) if _, ok := token.Method.(*jwt.SigningMethodRSA); !ok { return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"]) } return token.Header["kid"], nil } token, err := jwt.Parse(myToken, fc1) // No direct error comparison possible; since err is wrapped in another struct if err != nil && strings.Contains(err.Error(), jwt.ErrPEMMappingObsolete.Error()) { currentPEMsURL := "https://www.googleapis.com/oauth2/v1/certs" req, err := http.NewRequest("GET", currentPEMsURL, nil) if err != nil { lg("creation of pem request failed") return } req.Header.Set("Content-Type", "application/json") fo := fetch.Options{Req: req} fo.KnownProtocol = "https" fo.ForceHTTPSEvenOnDevelopmentServer = true bts, inf, err := fetch.UrlGetter(r, fo) lg(err) if err != nil { lg("tried to fetch %v, %v", currentPEMsURL, inf.URL) lg("msg %v", inf.Msg) return } if len(bts) > 200 { var data1 map[string]string err = json.Unmarshal(bts, &data1) lg(err) // lg(stringspb.IndentedDumpBytes(data1)) // w.Write(stringspb.IndentedDumpBytes(data1)) if len(data1) > 1 { lg("PEM mappings updated") jwt.MappingToPEM = data1 } else { lg("PEM mapping response contained only %v records; bytes length %v", len(data1), len(bts)) } } } token, err = jwt.Parse(myToken, fc1) if err != nil && strings.Contains(err.Error(), jwt.ErrInvalidKey.Error()) { w.Write([]byte("The submitted RSA Key was somehow unparseable. We still accept the token.\n")) /* https://developers.google.com/identity/sign-in/web/backend-auth */ err = nil token.Valid = true } if err != nil { w.Write([]byte("--- " + err.Error() + ".\n")) } if err == nil && token.Valid { tk := "" tk += fmt.Sprintf(" Algor: %v\n", token.Method) tk += fmt.Sprintf(" Header: %v\n", token.Header) for k, v := range token.Claims { tk += fmt.Sprintf("\t %-8v %v\n", k, v) } lg(tk) w.Write([]byte("tokensignin; valid. \n")) w.Write([]byte(tokSize)) sb := "header-sub-not-present" if _, ok := token.Claims["sub"]; ok { sb = token.Claims["sub"].(string) } w.Write([]byte("ID from PWT is " + sb + "\n")) _, usr, msg1 := login.CheckForNormalUser(r) if usr != nil { w.Write([]byte("ID from SRV is " + usr.ID + "\n")) } w.Write([]byte(msg1 + "\n")) } else { w.Write([]byte("tokensignin; INVALID. \n")) w.Write([]byte(tokSize)) w.Write([]byte(stringspb.ToLen(myToken, 30))) vrf := fmt.Sprintf("\nhttps://www.googleapis.com/oauth2/v3/tokeninfo?id_token=%v \n", myToken) w.Write([]byte(vrf)) } }
func fetchSimForm(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { lg, b := loghttp.BuffLoggerUniversal(w, r) closureOverBuf := func(bUnused *bytes.Buffer) { loghttp.Pf(w, r, b.String()) } defer closureOverBuf(b) // the argument is ignored, r.Header.Set("X-Custom-Header-Counter", "nocounter") // on live server => always use https if r.URL.Scheme != "https" && !util_appengine.IsLocalEnviron() { r.URL.Scheme = "https" r.URL.Host = r.Host lg("lo - redirect %v", r.URL.String()) http.Redirect(w, r, r.URL.String(), http.StatusFound) } err := r.ParseForm() lg(err) rURL := "" if r.FormValue(routes.URLParamKey) != "" { rURL = r.FormValue(routes.URLParamKey) } if len(rURL) == 0 { wpf(b, tplx.ExecTplHelper(tplx.Head, map[string]interface{}{"HtmlTitle": "Find similar HTML URLs"})) defer wpf(b, tplx.Foot) tm := map[string]string{ "val": "www.welt.de/politik/ausland/article146154432/Tuerkische-Bodentruppen-marschieren-im-Nordirak-ein.html", "fieldname": routes.URLParamKey, } tplForm := tt.Must(tt.New("tplName01").Parse(htmlForm)) tplForm.Execute(b, tm) } else { fullURL := fmt.Sprintf("https://%s%s?%s=%s&cnt=%s&prot=%s", r.Host, routes.FetchSimilarURI, routes.URLParamKey, rURL, r.FormValue("cnt"), r.FormValue("prot")) lg("lo - sending to URL 1: %v", fullURL) fo := fetch.Options{} fo.URL = fullURL bts, inf, err := fetch.UrlGetter(r, fo) _ = inf lg(err) if err != nil { return } if len(bts) == 0 { lg("empty bts") return } var mp map[string][]byte err = json.Unmarshal(bts, &mp) lg(err) if err != nil { lg("%s", bts) return } w.Header().Set("Content-Type", "text/html; charset=utf-8") if _, ok := mp["msg"]; ok { w.Write(mp["msg"]) } for k, v := range mp { if k != "msg" { wpf(w, "<br><br>%s:\n", k) if true { wpf(w, "len %v", len(v)) } else { wpf(w, "%s", html.EscapeString(string(v))) } } } } }