func init() { if util_appengine.IsLocalEnviron() { dns_router = "192.168.1.1" dns_cam = "192.168.1.4:8081" } else { dns_router = "ds7934.myfoscam.org" dns_cam = "ds7934.myfoscam.org:8081" } http.HandleFunc("/foscam-status", loghttp.Adapter(foscamStatus)) http.HandleFunc("/foscam-toggle", loghttp.Adapter(foscamToggle)) http.HandleFunc("/foscam-watch", loghttp.Adapter(foscamWatch)) }
func regroupFromDatastore01(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { c := appengine.NewContext(r) b1 := new(bytes.Buffer) defer func() { w.Header().Set("Content-Type", "text/html") w.Write(b1.Bytes()) }() var vVSrc [][]byte if util_appengine.IsLocalEnviron() { vVSrc = bq_statified_res1 } else { dsObj1, _ := dsu.BufGet(c, "dsu.WrapBlob__bq_res1") vVSrc = dsObj1.VVByte } if r.FormValue("mock") == "1" { dsObj1, _ := dsu.BufGet(c, "dsu.WrapBlob__bq_res_test") vVSrc = dsObj1.VVByte } var vVDest [][]byte = make([][]byte, len(vVSrc)) for i0 := 0; i0 < len(vVSrc); i0++ { s_row := string(vVSrc[i0]) v_row := stringspb.SplitByWhitespace(s_row) b_row := new(bytes.Buffer) b_row.WriteString(fmt.Sprintf("%16.12s ", v_row[3])) // leading spaces b_row.WriteString(fmt.Sprintf("%16.12s ", v_row[5])) b_row.WriteString(fmt.Sprintf("%16.8s", v_row[7])) vVDest[i0] = []byte(b_row.Bytes()) } key_combi, _ := dsu.BufPut(c, dsu.WrapBlob{Name: "res_processed_01", S: "[][]byte", VVByte: vVDest}, "res_processed_01") dsObj2, _ := dsu.BufGet(c, key_combi) printPlaintextTable(w, r, dsObj2.VVByte) }
// type Attribute struct { // Namespace, Key, Val string // } func rewriteAttributes(attributes []html.Attribute, proxyHostPort, remoteHost string) []html.Attribute { rew := make([]html.Attribute, 0, len(attributes)) for i := 0; i < len(attributes); i++ { attr := attributes[i] if attr.Key == "class" || attr.Key == "style" { continue } if attr.Key == "href" || attr.Key == "src" || attr.Key == "action" { // make absolute attr.Val = absolutize(attr.Val, remoteHost) } if attr.Key == "href" { attr.Val = fmt.Sprintf("%v?url=%v", routes.ProxifyURI, attr.Val) } if attr.Key == "src" { attr.Key = "href" } if attr.Key == "action" { // attr.Val = fmt.Sprintf("/blob2/form-redirector?redirect-to=%v", attr.Val) // appended as form field, thus not needed here if util_appengine.IsLocalEnviron() { attr.Val = fmt.Sprintf("http://%v%v", proxyHostPort, routes.FormRedirector) } else { attr.Val = fmt.Sprintf("https://%v%v", proxyHostPort, routes.FormRedirector) } } if attr.Key == "method" { attr.Val = "post" } rew = append(rew, attr) } rew = append(rew, html.Attribute{Key: "was", Val: "rewritten"}) rew = append(rew, html.Attribute{Key: "method", Val: "post"}) return rew }
func formRedirector(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { lg, lge := loghttp.Logger(w, r) var msg, cntnt, rURL string w.Header().Set("Content-type", "text/html; charset=utf-8") // w.Header().Set("Content-type", "text/html; charset=latin-1") rURL = r.FormValue("redirect-to") lg("lo redirect to: %v", rURL) if len(r.PostForm) > 0 { // loghttp.Pf(w, r, "post unimplemented:<br> %#v <br>\n", r.PostForm) // return msg += fmt.Sprintf("post converted to get<br>") } rURL = fmt.Sprintf("%v?1=2&", rURL) for key, vals := range r.Form { if key == "redirect-to" { continue } val := vals[0] if util_appengine.IsLocalEnviron() { val = strings.Replace(val, " ", "%20", -1) } rURL = fmt.Sprintf("%v&%v=%v", rURL, key, val) } bts, inf, err := fetch.UrlGetter(r, fetch.Options{URL: rURL}) lge(err) cntnt = string(bts) cntnt = insertNewlines.Replace(cntnt) cntnt = undouble.Replace(cntnt) cntnt = domclean1.ModifyHTML(r, inf.URL, cntnt) fmt.Fprintf(w, "%s \n\n", cntnt) fmt.Fprintf(w, "%s \n\n", msg) }
func fetchSimForm(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { lg, b := loghttp.BuffLoggerUniversal(w, r) closureOverBuf := func(bUnused *bytes.Buffer) { loghttp.Pf(w, r, b.String()) } defer closureOverBuf(b) // the argument is ignored, r.Header.Set("X-Custom-Header-Counter", "nocounter") // on live server => always use https if r.URL.Scheme != "https" && !util_appengine.IsLocalEnviron() { r.URL.Scheme = "https" r.URL.Host = r.Host lg("lo - redirect %v", r.URL.String()) http.Redirect(w, r, r.URL.String(), http.StatusFound) } err := r.ParseForm() lg(err) rURL := "" if r.FormValue(routes.URLParamKey) != "" { rURL = r.FormValue(routes.URLParamKey) } if len(rURL) == 0 { wpf(b, tplx.ExecTplHelper(tplx.Head, map[string]interface{}{"HtmlTitle": "Find similar HTML URLs"})) defer wpf(b, tplx.Foot) tm := map[string]string{ "val": "www.welt.de/politik/ausland/article146154432/Tuerkische-Bodentruppen-marschieren-im-Nordirak-ein.html", "fieldname": routes.URLParamKey, } tplForm := tt.Must(tt.New("tplName01").Parse(htmlForm)) tplForm.Execute(b, tm) } else { fullURL := fmt.Sprintf("https://%s%s?%s=%s&cnt=%s&prot=%s", r.Host, routes.FetchSimilarURI, routes.URLParamKey, rURL, r.FormValue("cnt"), r.FormValue("prot")) lg("lo - sending to URL 1: %v", fullURL) fo := fetch.Options{} fo.URL = fullURL bts, inf, err := fetch.UrlGetter(r, fo) _ = inf lg(err) if err != nil { return } if len(bts) == 0 { lg("empty bts") return } var mp map[string][]byte err = json.Unmarshal(bts, &mp) lg(err) if err != nil { lg("%s", bts) return } w.Header().Set("Content-Type", "text/html; charset=utf-8") if _, ok := mp["msg"]; ok { w.Write(mp["msg"]) } for k, v := range mp { if k != "msg" { wpf(w, "<br><br>%s:\n", k) if true { wpf(w, "len %v", len(v)) } else { wpf(w, "%s", html.EscapeString(string(v))) } } } } }
// Todo: When c==nil we are in a non-appengine environment. // We still want to return at least ii.PureHostname func GetByContext(c context.Context) *Instance { tstart := time.Now() if !ii.LastUpdated.IsZero() { age := tstart.Sub(ii.LastUpdated) if age < 200*time.Millisecond { aelog.Infof(c, "instance info update too recently: %v, skipping.\n", age) return ii } if age < 1*time.Hour { if len(ii.Hostname) > 2 { return ii } } aelog.Infof(c, "instance info update too old: %v, recomputing.\n", age) } ii.ModuleName = appengine.ModuleName(c) ii.InstanceID = appengine.InstanceID() ii.VersionFull = appengine.VersionID(c) majorMinor := strings.Split(ii.VersionFull, ".") if len(majorMinor) != 2 { panic("we need a version string of format X.Y") } ii.VersionMajor = majorMinor[0] ii.VersionMinor = majorMinor[1] var err = errors.New("dummy creation error message") ii.NumInstances, err = module.NumInstances(c, ii.ModuleName, ii.VersionFull) if err != nil { // this never works with version full // we do not log this - but try version major err = nil if !util_appengine.IsLocalEnviron() { ii.NumInstances, err = module.NumInstances(c, ii.ModuleName, ii.VersionMajor) if err != nil { eStr := err.Error() eCmp1, eCmp2, eCmp3 := "API error", "INVALID_VERSION)", "Could not find the given version" if strings.Contains(eStr, eCmp1) && strings.Contains(eStr, eCmp2) && strings.Contains(eStr, eCmp3) { aelog.Infof(c, "get num instances works only live and without autoscale; %v", err) } else { aelog.Errorf(c, "get num instances error; %v", err) } } } } // in auto scaling, google reports "zero" - which can not be true // we assume at least 1 if ii.NumInstances == 0 { ii.NumInstances = 1 } // http://[0-2].1.default.libertarian-islands.appspot.com/instance-info ii.Hostname, err = appengine.ModuleHostname(c, ii.ModuleName, ii.VersionMajor, "") if err != nil { aelog.Errorf(c, "ModuleHostname1: %v", err) } ii.PureHostname = appengine.DefaultVersionHostname(c) if !appengine.IsDevAppServer() { ii.HostnameInst0, err = appengine.ModuleHostname(c, ii.ModuleName, ii.VersionMajor, "0") if err != nil && (err.Error() == autoScalingErr1 || err.Error() == autoScalingErr2) { aelog.Infof(c, "inst 0: "+autoScalingErrMsg) err = nil } if err != nil { aelog.Errorf(c, "ModuleHostname2: %v", err) } ii.HostnameInst1, err = appengine.ModuleHostname(c, ii.ModuleName, ii.VersionMajor, "1") if err != nil && (err.Error() == autoScalingErr1 || err.Error() == autoScalingErr2) { aelog.Infof(c, "inst 1: "+autoScalingErrMsg) err = nil } if err != nil { aelog.Errorf(c, "ModuleHostname3: %v", err) } ii.HostnameMod02, err = appengine.ModuleHostname(c, "mod02", "", "") if err != nil { aelog.Infof(c, "ModuleHostname4: %v", err) } } ii.LastUpdated = time.Now() aelog.Infof(c, "collectInfo() completed, %v - %v - %v - %v - %v, took %v", stringspb.Ellipsoider(ii.InstanceID, 4), ii.VersionMajor, ii.ModuleName, ii.Hostname, ii.PureHostname, time.Now().Sub(tstart)) return ii }
func attrsAbsoluteAndProxified(attributes []html.Attribute, proxyHostPort, remoteHost string) []html.Attribute { rew := make([]html.Attribute, 0, len(attributes)) for i := 0; i < len(attributes); i++ { attr := attributes[i] // Make all absolute if attr.Key == "href" || attr.Key == "src" || attr.Key == "action" { // make absolute if attrX(attributes, "cfrom") == "img" { attr.Val = absolutize("http", remoteHost, attr.Val) } else { attr.Val = absolutize("https", remoteHost, attr.Val) } } if attr.Key == "href" { if attrX(attributes, "cfrom") == "img" { // dont proxify image links, except they are relative // we already absolutized them above } else { // proxify - v1 // no rewrite to https-posted-form; urls remain visible if false { // only proxy attr.Val = fmt.Sprintf("%v?%v=%v", routes.ProxifyURI, routes.URLParamKey, attr.Val) } else { // dedup attr.Val = fmt.Sprintf("%v?%v=%v", routes.DedupURI, routes.URLParamKey, attr.Val) } if util_appengine.IsLocalEnviron() { attr.Val = fmt.Sprintf("http://%v%v", proxyHostPort, attr.Val) } else { attr.Val = fmt.Sprintf("https://%v%v", proxyHostPort, attr.Val) } } } if attr.Key == "action" { // proxify - v2 // Since we appended a form field, we do not need: // action = spf("/blob2/form-redirector?redirect-to=%v", action) if util_appengine.IsLocalEnviron() { attr.Val = fmt.Sprintf("http://%v%v", proxyHostPort, routes.FormRedirector) } else { attr.Val = fmt.Sprintf("https://%v%v", proxyHostPort, routes.FormRedirector) } } rew = append(rew, attr) } // // // We instrumented all forms with a field "redirect-to" // Now we have to make the value of this field absolute isRedirectInput := false for _, attr := range rew { if attr.Key == "name" && attr.Val == "redirect-to" { isRedirectInput = true } } if isRedirectInput { for _, attr := range rew { if attr.Key == "value" { attr.Val = absolutize("https", remoteHost, attr.Val) } } } return rew }
// handleFetchURL either displays a form for requesting an url // or it returns the URL´s contents. func handleFetchURL(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { lg, b := loghttp.BuffLoggerUniversal(w, r) _ = b // on live server => always use https if r.URL.Scheme != "https" && !util_appengine.IsLocalEnviron() { r.URL.Scheme = "https" r.URL.Host = r.Host lg("lo - redirect %v", r.URL.String()) http.Redirect(w, r, r.URL.String(), http.StatusFound) } /* To distinguish between posted and getted value, we check the "post-only" slice of values first. If nothing's there, but FormValue *has* a value, then it was "getted", otherwise "posted" */ rURL := "" urlAs := "" err := r.ParseForm() lg(err) if r.PostFormValue(routes.URLParamKey) != "" { urlAs += "url posted " rURL = r.PostFormValue(routes.URLParamKey) } if r.FormValue(routes.URLParamKey) != "" { if rURL == "" { urlAs += "url getted " rURL = r.FormValue(routes.URLParamKey) } } // lg("received %v: %q", urlAs, rURL) if len(rURL) == 0 { tplAdder, tplExec := tplx.FuncTplBuilder(w, r) tplAdder("n_html_title", "Fetch some http data", nil) m := map[string]string{ "protocol": "https", "host": r.Host, // not fetch.HostFromReq(r) "path": routes.ProxifyURI, "name": routes.URLParamKey, "val": "google.com", } if util_appengine.IsLocalEnviron() { m["protocol"] = "http" } tplAdder("n_cont_0", c_formFetchUrl, m) tplExec(w, r) } else { r.Header.Set("X-Custom-Header-Counter", "nocounter") bts, inf, err := fetch.UrlGetter(r, fetch.Options{URL: rURL}) lg(err) tp := mime.TypeByExtension(path.Ext(inf.URL.Path)) if false { ext := path.Ext(rURL) ext = strings.ToLower(ext) tp = mime.TypeByExtension(ext) } w.Header().Set("Content-Type", tp) // w.Header().Set("Content-type", "text/html; charset=latin-1") if r.FormValue("dbg") != "" { w.Header().Set("Content-type", "text/html; charset=utf-8") fmt.Fprintf(w, "%s<br>\n %s<br>\n %v", inf.URL.Path, tp, inf.URL.String()) return } opts := domclean2.CleaningOptions{Proxify: true} opts.Beautify = true // "<a> Linktext without trailing space" opts.RemoteHost = fetch.HostFromStringUrl(rURL) // opts.ProxyHost = routes.AppHost() opts.ProxyHost = fetch.HostFromReq(r) if !util_appengine.IsLocalEnviron() { opts.ProxyHost = fetch.HostFromReq(r) } doc, err := domclean2.DomClean(bts, opts) var bufRend bytes.Buffer err = html.Render(&bufRend, doc) lg(err) w.Write(bufRend.Bytes()) } }