Beispiel #1
0
func init() {

	if util_appengine.IsLocalEnviron() {
		dns_router = "192.168.1.1"
		dns_cam = "192.168.1.4:8081"
	} else {
		dns_router = "ds7934.myfoscam.org"
		dns_cam = "ds7934.myfoscam.org:8081"
	}

	http.HandleFunc("/foscam-status", loghttp.Adapter(foscamStatus))
	http.HandleFunc("/foscam-toggle", loghttp.Adapter(foscamToggle))
	http.HandleFunc("/foscam-watch", loghttp.Adapter(foscamWatch))

}
Beispiel #2
0
func regroupFromDatastore01(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {

	c := appengine.NewContext(r)

	b1 := new(bytes.Buffer)
	defer func() {
		w.Header().Set("Content-Type", "text/html")
		w.Write(b1.Bytes())
	}()

	var vVSrc [][]byte

	if util_appengine.IsLocalEnviron() {
		vVSrc = bq_statified_res1
	} else {
		dsObj1, _ := dsu.BufGet(c, "dsu.WrapBlob__bq_res1")
		vVSrc = dsObj1.VVByte
	}

	if r.FormValue("mock") == "1" {
		dsObj1, _ := dsu.BufGet(c, "dsu.WrapBlob__bq_res_test")
		vVSrc = dsObj1.VVByte
	}

	var vVDest [][]byte = make([][]byte, len(vVSrc))

	for i0 := 0; i0 < len(vVSrc); i0++ {

		s_row := string(vVSrc[i0])
		v_row := stringspb.SplitByWhitespace(s_row)
		b_row := new(bytes.Buffer)

		b_row.WriteString(fmt.Sprintf("%16.12s   ", v_row[3])) // leading spaces
		b_row.WriteString(fmt.Sprintf("%16.12s   ", v_row[5]))
		b_row.WriteString(fmt.Sprintf("%16.8s", v_row[7]))

		vVDest[i0] = []byte(b_row.Bytes())

	}

	key_combi, _ := dsu.BufPut(c, dsu.WrapBlob{Name: "res_processed_01", S: "[][]byte", VVByte: vVDest}, "res_processed_01")
	dsObj2, _ := dsu.BufGet(c, key_combi)

	printPlaintextTable(w, r, dsObj2.VVByte)

}
Beispiel #3
0
// type Attribute struct {
//     Namespace, Key, Val string
// }
func rewriteAttributes(attributes []html.Attribute, proxyHostPort, remoteHost string) []html.Attribute {

	rew := make([]html.Attribute, 0, len(attributes))

	for i := 0; i < len(attributes); i++ {
		attr := attributes[i]

		if attr.Key == "class" || attr.Key == "style" {
			continue
		}

		if attr.Key == "href" || attr.Key == "src" || attr.Key == "action" { //  make absolute
			attr.Val = absolutize(attr.Val, remoteHost)
		}

		if attr.Key == "href" {
			attr.Val = fmt.Sprintf("%v?url=%v", routes.ProxifyURI, attr.Val)
		}

		if attr.Key == "src" {
			attr.Key = "href"
		}

		if attr.Key == "action" {
			// attr.Val = fmt.Sprintf("/blob2/form-redirector?redirect-to=%v", attr.Val) // appended as form field, thus not needed here
			if util_appengine.IsLocalEnviron() {
				attr.Val = fmt.Sprintf("http://%v%v", proxyHostPort, routes.FormRedirector)
			} else {
				attr.Val = fmt.Sprintf("https://%v%v", proxyHostPort, routes.FormRedirector)
			}

		}

		if attr.Key == "method" {
			attr.Val = "post"
		}

		rew = append(rew, attr)
	}

	rew = append(rew, html.Attribute{Key: "was", Val: "rewritten"})
	rew = append(rew, html.Attribute{Key: "method", Val: "post"})

	return rew
}
Beispiel #4
0
func formRedirector(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {

	lg, lge := loghttp.Logger(w, r)

	var msg, cntnt, rURL string

	w.Header().Set("Content-type", "text/html; charset=utf-8")
	// w.Header().Set("Content-type", "text/html; charset=latin-1")

	rURL = r.FormValue("redirect-to")
	lg("lo redirect to: %v", rURL)

	if len(r.PostForm) > 0 {
		// loghttp.Pf(w, r, "post unimplemented:<br> %#v <br>\n", r.PostForm)
		// return
		msg += fmt.Sprintf("post converted to get<br>")
	}

	rURL = fmt.Sprintf("%v?1=2&", rURL)
	for key, vals := range r.Form {
		if key == "redirect-to" {
			continue
		}
		val := vals[0]
		if util_appengine.IsLocalEnviron() {
			val = strings.Replace(val, " ", "%20", -1)
		}
		rURL = fmt.Sprintf("%v&%v=%v", rURL, key, val)
	}

	bts, inf, err := fetch.UrlGetter(r, fetch.Options{URL: rURL})
	lge(err)

	cntnt = string(bts)

	cntnt = insertNewlines.Replace(cntnt)
	cntnt = undouble.Replace(cntnt)

	cntnt = domclean1.ModifyHTML(r, inf.URL, cntnt)

	fmt.Fprintf(w, "%s \n\n", cntnt)
	fmt.Fprintf(w, "%s \n\n", msg)

}
func fetchSimForm(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {

	lg, b := loghttp.BuffLoggerUniversal(w, r)
	closureOverBuf := func(bUnused *bytes.Buffer) {
		loghttp.Pf(w, r, b.String())
	}
	defer closureOverBuf(b) // the argument is ignored,

	r.Header.Set("X-Custom-Header-Counter", "nocounter")

	// on live server => always use https
	if r.URL.Scheme != "https" && !util_appengine.IsLocalEnviron() {
		r.URL.Scheme = "https"
		r.URL.Host = r.Host
		lg("lo - redirect %v", r.URL.String())
		http.Redirect(w, r, r.URL.String(), http.StatusFound)
	}

	err := r.ParseForm()
	lg(err)

	rURL := ""
	if r.FormValue(routes.URLParamKey) != "" {
		rURL = r.FormValue(routes.URLParamKey)
	}
	if len(rURL) == 0 {

		wpf(b, tplx.ExecTplHelper(tplx.Head, map[string]interface{}{"HtmlTitle": "Find similar HTML URLs"}))
		defer wpf(b, tplx.Foot)

		tm := map[string]string{
			"val":       "www.welt.de/politik/ausland/article146154432/Tuerkische-Bodentruppen-marschieren-im-Nordirak-ein.html",
			"fieldname": routes.URLParamKey,
		}
		tplForm := tt.Must(tt.New("tplName01").Parse(htmlForm))
		tplForm.Execute(b, tm)

	} else {

		fullURL := fmt.Sprintf("https://%s%s?%s=%s&cnt=%s&prot=%s", r.Host, routes.FetchSimilarURI,
			routes.URLParamKey, rURL, r.FormValue("cnt"), r.FormValue("prot"))
		lg("lo - sending to URL 1: %v", fullURL)

		fo := fetch.Options{}
		fo.URL = fullURL
		bts, inf, err := fetch.UrlGetter(r, fo)
		_ = inf
		lg(err)
		if err != nil {
			return
		}

		if len(bts) == 0 {
			lg("empty bts")
			return
		}

		var mp map[string][]byte
		err = json.Unmarshal(bts, &mp)
		lg(err)
		if err != nil {
			lg("%s", bts)
			return
		}

		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		if _, ok := mp["msg"]; ok {
			w.Write(mp["msg"])
		}

		for k, v := range mp {
			if k != "msg" {
				wpf(w, "<br><br>%s:\n", k)
				if true {
					wpf(w, "len %v", len(v))
				} else {
					wpf(w, "%s", html.EscapeString(string(v)))
				}
			}
		}

	}

}
Beispiel #6
0
// Todo: When c==nil we are in a non-appengine environment.
// We still want to return at least ii.PureHostname
func GetByContext(c context.Context) *Instance {

	tstart := time.Now()

	if !ii.LastUpdated.IsZero() {

		age := tstart.Sub(ii.LastUpdated)

		if age < 200*time.Millisecond {
			aelog.Infof(c, "instance info update too recently: %v, skipping.\n", age)
			return ii
		}

		if age < 1*time.Hour {
			if len(ii.Hostname) > 2 {
				return ii
			}

		}

		aelog.Infof(c, "instance info update too old: %v, recomputing.\n", age)
	}

	ii.ModuleName = appengine.ModuleName(c)
	ii.InstanceID = appengine.InstanceID()
	ii.VersionFull = appengine.VersionID(c)

	majorMinor := strings.Split(ii.VersionFull, ".")
	if len(majorMinor) != 2 {
		panic("we need a version string of format X.Y")
	}

	ii.VersionMajor = majorMinor[0]
	ii.VersionMinor = majorMinor[1]

	var err = errors.New("dummy creation error message")

	ii.NumInstances, err = module.NumInstances(c, ii.ModuleName, ii.VersionFull)
	if err != nil {
		// this never works with version full
		// we do not log this - but try version major
		err = nil

		if !util_appengine.IsLocalEnviron() {
			ii.NumInstances, err = module.NumInstances(c, ii.ModuleName, ii.VersionMajor)

			if err != nil {
				eStr := err.Error()
				eCmp1, eCmp2, eCmp3 := "API error", "INVALID_VERSION)", "Could not find the given version"
				if strings.Contains(eStr, eCmp1) && strings.Contains(eStr, eCmp2) && strings.Contains(eStr, eCmp3) {
					aelog.Infof(c, "get num instances works only live and without autoscale; %v", err)
				} else {
					aelog.Errorf(c, "get num instances error; %v", err)
				}
			}

		}

	}

	// in auto scaling, google reports "zero" - which can not be true
	// we assume at least 1
	if ii.NumInstances == 0 {
		ii.NumInstances = 1
	}

	// http://[0-2].1.default.libertarian-islands.appspot.com/instance-info

	ii.Hostname, err = appengine.ModuleHostname(c, ii.ModuleName, ii.VersionMajor, "")
	if err != nil {
		aelog.Errorf(c, "ModuleHostname1: %v", err)
	}

	ii.PureHostname = appengine.DefaultVersionHostname(c)

	if !appengine.IsDevAppServer() {
		ii.HostnameInst0, err = appengine.ModuleHostname(c, ii.ModuleName, ii.VersionMajor, "0")
		if err != nil && (err.Error() == autoScalingErr1 || err.Error() == autoScalingErr2) {
			aelog.Infof(c, "inst 0: "+autoScalingErrMsg)
			err = nil
		}
		if err != nil {
			aelog.Errorf(c, "ModuleHostname2: %v", err)
		}

		ii.HostnameInst1, err = appengine.ModuleHostname(c, ii.ModuleName, ii.VersionMajor, "1")
		if err != nil && (err.Error() == autoScalingErr1 || err.Error() == autoScalingErr2) {
			aelog.Infof(c, "inst 1: "+autoScalingErrMsg)
			err = nil
		}
		if err != nil {
			aelog.Errorf(c, "ModuleHostname3: %v", err)
		}

		ii.HostnameMod02, err = appengine.ModuleHostname(c, "mod02", "", "")
		if err != nil {
			aelog.Infof(c, "ModuleHostname4: %v", err)
		}

	}

	ii.LastUpdated = time.Now()

	aelog.Infof(c, "collectInfo() completed, %v  - %v - %v - %v - %v, took %v",
		stringspb.Ellipsoider(ii.InstanceID, 4), ii.VersionMajor, ii.ModuleName,
		ii.Hostname, ii.PureHostname, time.Now().Sub(tstart))

	return ii
}
Beispiel #7
0
func attrsAbsoluteAndProxified(attributes []html.Attribute, proxyHostPort, remoteHost string) []html.Attribute {

	rew := make([]html.Attribute, 0, len(attributes))

	for i := 0; i < len(attributes); i++ {

		attr := attributes[i]

		// Make all absolute
		if attr.Key == "href" || attr.Key == "src" || attr.Key == "action" { //  make absolute

			if attrX(attributes, "cfrom") == "img" {
				attr.Val = absolutize("http", remoteHost, attr.Val)
			} else {
				attr.Val = absolutize("https", remoteHost, attr.Val)
			}

		}

		if attr.Key == "href" {

			if attrX(attributes, "cfrom") == "img" {
				// dont proxify image links, except they are relative
				// we already absolutized them  above

			} else {
				// proxify - v1
				// no rewrite to https-posted-form; urls remain visible
				if false {
					// only proxy
					attr.Val = fmt.Sprintf("%v?%v=%v", routes.ProxifyURI, routes.URLParamKey, attr.Val)
				} else {
					// dedup
					attr.Val = fmt.Sprintf("%v?%v=%v", routes.DedupURI, routes.URLParamKey, attr.Val)
				}

				if util_appengine.IsLocalEnviron() {
					attr.Val = fmt.Sprintf("http://%v%v", proxyHostPort, attr.Val)
				} else {
					attr.Val = fmt.Sprintf("https://%v%v", proxyHostPort, attr.Val)
				}
			}

		}

		if attr.Key == "action" {

			// proxify - v2

			// Since we appended a form field, we do not need:
			// action = spf("/blob2/form-redirector?redirect-to=%v", action)
			if util_appengine.IsLocalEnviron() {
				attr.Val = fmt.Sprintf("http://%v%v", proxyHostPort, routes.FormRedirector)
			} else {
				attr.Val = fmt.Sprintf("https://%v%v", proxyHostPort, routes.FormRedirector)
			}

		}

		rew = append(rew, attr)
	}

	//
	//
	// We instrumented all forms with a field "redirect-to"
	// Now we have to make the value of this field absolute
	isRedirectInput := false
	for _, attr := range rew {
		if attr.Key == "name" && attr.Val == "redirect-to" {
			isRedirectInput = true
		}
	}
	if isRedirectInput {
		for _, attr := range rew {
			if attr.Key == "value" {
				attr.Val = absolutize("https", remoteHost, attr.Val)
			}
		}
	}

	return rew
}
Beispiel #8
0
// handleFetchURL either displays a form for requesting an url
// or it returns the URL´s contents.
func handleFetchURL(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {

	lg, b := loghttp.BuffLoggerUniversal(w, r)
	_ = b

	// on live server => always use https
	if r.URL.Scheme != "https" && !util_appengine.IsLocalEnviron() {
		r.URL.Scheme = "https"
		r.URL.Host = r.Host
		lg("lo - redirect %v", r.URL.String())
		http.Redirect(w, r, r.URL.String(), http.StatusFound)
	}

	/*
		To distinguish between posted and getted value,
		we check the "post-only" slice of values first.
		If nothing's there, but FormValue *has* a value,
		then it was "getted", otherwise "posted"
	*/
	rURL := ""
	urlAs := ""
	err := r.ParseForm()
	lg(err)
	if r.PostFormValue(routes.URLParamKey) != "" {
		urlAs += "url posted "
		rURL = r.PostFormValue(routes.URLParamKey)
	}

	if r.FormValue(routes.URLParamKey) != "" {
		if rURL == "" {
			urlAs += "url getted "
			rURL = r.FormValue(routes.URLParamKey)
		}
	}
	// lg("received %v:  %q", urlAs, rURL)

	if len(rURL) == 0 {

		tplAdder, tplExec := tplx.FuncTplBuilder(w, r)
		tplAdder("n_html_title", "Fetch some http data", nil)

		m := map[string]string{
			"protocol": "https",
			"host":     r.Host, // not  fetch.HostFromReq(r)
			"path":     routes.ProxifyURI,
			"name":     routes.URLParamKey,
			"val":      "google.com",
		}
		if util_appengine.IsLocalEnviron() {
			m["protocol"] = "http"
		}
		tplAdder("n_cont_0", c_formFetchUrl, m)
		tplExec(w, r)

	} else {

		r.Header.Set("X-Custom-Header-Counter", "nocounter")

		bts, inf, err := fetch.UrlGetter(r, fetch.Options{URL: rURL})
		lg(err)

		tp := mime.TypeByExtension(path.Ext(inf.URL.Path))
		if false {
			ext := path.Ext(rURL)
			ext = strings.ToLower(ext)
			tp = mime.TypeByExtension(ext)
		}
		w.Header().Set("Content-Type", tp)
		// w.Header().Set("Content-type", "text/html; charset=latin-1")

		if r.FormValue("dbg") != "" {
			w.Header().Set("Content-type", "text/html; charset=utf-8")
			fmt.Fprintf(w, "%s<br>\n  %s<br>\n %v", inf.URL.Path, tp, inf.URL.String())
			return
		}

		opts := domclean2.CleaningOptions{Proxify: true}
		opts.Beautify = true // "<a> Linktext without trailing space"
		opts.RemoteHost = fetch.HostFromStringUrl(rURL)

		// opts.ProxyHost = routes.AppHost()
		opts.ProxyHost = fetch.HostFromReq(r)
		if !util_appengine.IsLocalEnviron() {
			opts.ProxyHost = fetch.HostFromReq(r)
		}

		doc, err := domclean2.DomClean(bts, opts)

		var bufRend bytes.Buffer
		err = html.Render(&bufRend, doc)
		lg(err)
		w.Write(bufRend.Bytes())

	}

}