func init() {
if util_appengine.IsLocalEnviron() {
dns_router = "192.168.1.1"
dns_cam = "192.168.1.4:8081"
} else {
dns_router = "ds7934.myfoscam.org"
dns_cam = "ds7934.myfoscam.org:8081"
}
http.HandleFunc("/foscam-status", loghttp.Adapter(foscamStatus))
http.HandleFunc("/foscam-toggle", loghttp.Adapter(foscamToggle))
http.HandleFunc("/foscam-watch", loghttp.Adapter(foscamWatch))
}
func regroupFromDatastore01(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {
c := appengine.NewContext(r)
b1 := new(bytes.Buffer)
defer func() {
w.Header().Set("Content-Type", "text/html")
w.Write(b1.Bytes())
}()
var vVSrc [][]byte
if util_appengine.IsLocalEnviron() {
vVSrc = bq_statified_res1
} else {
dsObj1, _ := dsu.BufGet(c, "dsu.WrapBlob__bq_res1")
vVSrc = dsObj1.VVByte
}
if r.FormValue("mock") == "1" {
dsObj1, _ := dsu.BufGet(c, "dsu.WrapBlob__bq_res_test")
vVSrc = dsObj1.VVByte
}
var vVDest [][]byte = make([][]byte, len(vVSrc))
for i0 := 0; i0 < len(vVSrc); i0++ {
s_row := string(vVSrc[i0])
v_row := stringspb.SplitByWhitespace(s_row)
b_row := new(bytes.Buffer)
b_row.WriteString(fmt.Sprintf("%16.12s ", v_row[3])) // leading spaces
b_row.WriteString(fmt.Sprintf("%16.12s ", v_row[5]))
b_row.WriteString(fmt.Sprintf("%16.8s", v_row[7]))
vVDest[i0] = []byte(b_row.Bytes())
}
key_combi, _ := dsu.BufPut(c, dsu.WrapBlob{Name: "res_processed_01", S: "[][]byte", VVByte: vVDest}, "res_processed_01")
dsObj2, _ := dsu.BufGet(c, key_combi)
printPlaintextTable(w, r, dsObj2.VVByte)
}
// type Attribute struct {
// Namespace, Key, Val string
// }
func rewriteAttributes(attributes []html.Attribute, proxyHostPort, remoteHost string) []html.Attribute {
rew := make([]html.Attribute, 0, len(attributes))
for i := 0; i < len(attributes); i++ {
attr := attributes[i]
if attr.Key == "class" || attr.Key == "style" {
continue
}
if attr.Key == "href" || attr.Key == "src" || attr.Key == "action" { // make absolute
attr.Val = absolutize(attr.Val, remoteHost)
}
if attr.Key == "href" {
attr.Val = fmt.Sprintf("%v?url=%v", routes.ProxifyURI, attr.Val)
}
if attr.Key == "src" {
attr.Key = "href"
}
if attr.Key == "action" {
// attr.Val = fmt.Sprintf("/blob2/form-redirector?redirect-to=%v", attr.Val) // appended as form field, thus not needed here
if util_appengine.IsLocalEnviron() {
attr.Val = fmt.Sprintf("http://%v%v", proxyHostPort, routes.FormRedirector)
} else {
attr.Val = fmt.Sprintf("https://%v%v", proxyHostPort, routes.FormRedirector)
}
}
if attr.Key == "method" {
attr.Val = "post"
}
rew = append(rew, attr)
}
rew = append(rew, html.Attribute{Key: "was", Val: "rewritten"})
rew = append(rew, html.Attribute{Key: "method", Val: "post"})
return rew
}
func formRedirector(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {
lg, lge := loghttp.Logger(w, r)
var msg, cntnt, rURL string
w.Header().Set("Content-type", "text/html; charset=utf-8")
// w.Header().Set("Content-type", "text/html; charset=latin-1")
rURL = r.FormValue("redirect-to")
lg("lo redirect to: %v", rURL)
if len(r.PostForm) > 0 {
// loghttp.Pf(w, r, "post unimplemented:<br> %#v <br>\n", r.PostForm)
// return
msg += fmt.Sprintf("post converted to get<br>")
}
rURL = fmt.Sprintf("%v?1=2&", rURL)
for key, vals := range r.Form {
if key == "redirect-to" {
continue
}
val := vals[0]
if util_appengine.IsLocalEnviron() {
val = strings.Replace(val, " ", "%20", -1)
}
rURL = fmt.Sprintf("%v&%v=%v", rURL, key, val)
}
bts, inf, err := fetch.UrlGetter(r, fetch.Options{URL: rURL})
lge(err)
cntnt = string(bts)
cntnt = insertNewlines.Replace(cntnt)
cntnt = undouble.Replace(cntnt)
cntnt = domclean1.ModifyHTML(r, inf.URL, cntnt)
fmt.Fprintf(w, "%s \n\n", cntnt)
fmt.Fprintf(w, "%s \n\n", msg)
}
func fetchSimForm(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {
lg, b := loghttp.BuffLoggerUniversal(w, r)
closureOverBuf := func(bUnused *bytes.Buffer) {
loghttp.Pf(w, r, b.String())
}
defer closureOverBuf(b) // the argument is ignored,
r.Header.Set("X-Custom-Header-Counter", "nocounter")
// on live server => always use https
if r.URL.Scheme != "https" && !util_appengine.IsLocalEnviron() {
r.URL.Scheme = "https"
r.URL.Host = r.Host
lg("lo - redirect %v", r.URL.String())
http.Redirect(w, r, r.URL.String(), http.StatusFound)
}
err := r.ParseForm()
lg(err)
rURL := ""
if r.FormValue(routes.URLParamKey) != "" {
rURL = r.FormValue(routes.URLParamKey)
}
if len(rURL) == 0 {
wpf(b, tplx.ExecTplHelper(tplx.Head, map[string]interface{}{"HtmlTitle": "Find similar HTML URLs"}))
defer wpf(b, tplx.Foot)
tm := map[string]string{
"val": "www.welt.de/politik/ausland/article146154432/Tuerkische-Bodentruppen-marschieren-im-Nordirak-ein.html",
"fieldname": routes.URLParamKey,
}
tplForm := tt.Must(tt.New("tplName01").Parse(htmlForm))
tplForm.Execute(b, tm)
} else {
fullURL := fmt.Sprintf("https://%s%s?%s=%s&cnt=%s&prot=%s", r.Host, routes.FetchSimilarURI,
routes.URLParamKey, rURL, r.FormValue("cnt"), r.FormValue("prot"))
lg("lo - sending to URL 1: %v", fullURL)
fo := fetch.Options{}
fo.URL = fullURL
bts, inf, err := fetch.UrlGetter(r, fo)
_ = inf
lg(err)
if err != nil {
return
}
if len(bts) == 0 {
lg("empty bts")
return
}
var mp map[string][]byte
err = json.Unmarshal(bts, &mp)
lg(err)
if err != nil {
lg("%s", bts)
return
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
if _, ok := mp["msg"]; ok {
w.Write(mp["msg"])
}
for k, v := range mp {
if k != "msg" {
wpf(w, "<br><br>%s:\n", k)
if true {
wpf(w, "len %v", len(v))
} else {
wpf(w, "%s", html.EscapeString(string(v)))
}
}
}
}
}
// Todo: When c==nil we are in a non-appengine environment.
// We still want to return at least ii.PureHostname
func GetByContext(c context.Context) *Instance {
tstart := time.Now()
if !ii.LastUpdated.IsZero() {
age := tstart.Sub(ii.LastUpdated)
if age < 200*time.Millisecond {
aelog.Infof(c, "instance info update too recently: %v, skipping.\n", age)
return ii
}
if age < 1*time.Hour {
if len(ii.Hostname) > 2 {
return ii
}
}
aelog.Infof(c, "instance info update too old: %v, recomputing.\n", age)
}
ii.ModuleName = appengine.ModuleName(c)
ii.InstanceID = appengine.InstanceID()
ii.VersionFull = appengine.VersionID(c)
majorMinor := strings.Split(ii.VersionFull, ".")
if len(majorMinor) != 2 {
panic("we need a version string of format X.Y")
}
ii.VersionMajor = majorMinor[0]
ii.VersionMinor = majorMinor[1]
var err = errors.New("dummy creation error message")
ii.NumInstances, err = module.NumInstances(c, ii.ModuleName, ii.VersionFull)
if err != nil {
// this never works with version full
// we do not log this - but try version major
err = nil
if !util_appengine.IsLocalEnviron() {
ii.NumInstances, err = module.NumInstances(c, ii.ModuleName, ii.VersionMajor)
if err != nil {
eStr := err.Error()
eCmp1, eCmp2, eCmp3 := "API error", "INVALID_VERSION)", "Could not find the given version"
if strings.Contains(eStr, eCmp1) && strings.Contains(eStr, eCmp2) && strings.Contains(eStr, eCmp3) {
aelog.Infof(c, "get num instances works only live and without autoscale; %v", err)
} else {
aelog.Errorf(c, "get num instances error; %v", err)
}
}
}
}
// in auto scaling, google reports "zero" - which can not be true
// we assume at least 1
if ii.NumInstances == 0 {
ii.NumInstances = 1
}
// http://[0-2].1.default.libertarian-islands.appspot.com/instance-info
ii.Hostname, err = appengine.ModuleHostname(c, ii.ModuleName, ii.VersionMajor, "")
if err != nil {
aelog.Errorf(c, "ModuleHostname1: %v", err)
}
ii.PureHostname = appengine.DefaultVersionHostname(c)
if !appengine.IsDevAppServer() {
ii.HostnameInst0, err = appengine.ModuleHostname(c, ii.ModuleName, ii.VersionMajor, "0")
if err != nil && (err.Error() == autoScalingErr1 || err.Error() == autoScalingErr2) {
aelog.Infof(c, "inst 0: "+autoScalingErrMsg)
err = nil
}
if err != nil {
aelog.Errorf(c, "ModuleHostname2: %v", err)
}
ii.HostnameInst1, err = appengine.ModuleHostname(c, ii.ModuleName, ii.VersionMajor, "1")
if err != nil && (err.Error() == autoScalingErr1 || err.Error() == autoScalingErr2) {
aelog.Infof(c, "inst 1: "+autoScalingErrMsg)
err = nil
}
if err != nil {
aelog.Errorf(c, "ModuleHostname3: %v", err)
}
ii.HostnameMod02, err = appengine.ModuleHostname(c, "mod02", "", "")
if err != nil {
aelog.Infof(c, "ModuleHostname4: %v", err)
}
}
ii.LastUpdated = time.Now()
aelog.Infof(c, "collectInfo() completed, %v - %v - %v - %v - %v, took %v",
stringspb.Ellipsoider(ii.InstanceID, 4), ii.VersionMajor, ii.ModuleName,
ii.Hostname, ii.PureHostname, time.Now().Sub(tstart))
return ii
}
func attrsAbsoluteAndProxified(attributes []html.Attribute, proxyHostPort, remoteHost string) []html.Attribute {
rew := make([]html.Attribute, 0, len(attributes))
for i := 0; i < len(attributes); i++ {
attr := attributes[i]
// Make all absolute
if attr.Key == "href" || attr.Key == "src" || attr.Key == "action" { // make absolute
if attrX(attributes, "cfrom") == "img" {
attr.Val = absolutize("http", remoteHost, attr.Val)
} else {
attr.Val = absolutize("https", remoteHost, attr.Val)
}
}
if attr.Key == "href" {
if attrX(attributes, "cfrom") == "img" {
// dont proxify image links, except they are relative
// we already absolutized them above
} else {
// proxify - v1
// no rewrite to https-posted-form; urls remain visible
if false {
// only proxy
attr.Val = fmt.Sprintf("%v?%v=%v", routes.ProxifyURI, routes.URLParamKey, attr.Val)
} else {
// dedup
attr.Val = fmt.Sprintf("%v?%v=%v", routes.DedupURI, routes.URLParamKey, attr.Val)
}
if util_appengine.IsLocalEnviron() {
attr.Val = fmt.Sprintf("http://%v%v", proxyHostPort, attr.Val)
} else {
attr.Val = fmt.Sprintf("https://%v%v", proxyHostPort, attr.Val)
}
}
}
if attr.Key == "action" {
// proxify - v2
// Since we appended a form field, we do not need:
// action = spf("/blob2/form-redirector?redirect-to=%v", action)
if util_appengine.IsLocalEnviron() {
attr.Val = fmt.Sprintf("http://%v%v", proxyHostPort, routes.FormRedirector)
} else {
attr.Val = fmt.Sprintf("https://%v%v", proxyHostPort, routes.FormRedirector)
}
}
rew = append(rew, attr)
}
//
//
// We instrumented all forms with a field "redirect-to"
// Now we have to make the value of this field absolute
isRedirectInput := false
for _, attr := range rew {
if attr.Key == "name" && attr.Val == "redirect-to" {
isRedirectInput = true
}
}
if isRedirectInput {
for _, attr := range rew {
if attr.Key == "value" {
attr.Val = absolutize("https", remoteHost, attr.Val)
}
}
}
return rew
}
// handleFetchURL either displays a form for requesting an url
// or it returns the URL´s contents.
func handleFetchURL(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {
lg, b := loghttp.BuffLoggerUniversal(w, r)
_ = b
// on live server => always use https
if r.URL.Scheme != "https" && !util_appengine.IsLocalEnviron() {
r.URL.Scheme = "https"
r.URL.Host = r.Host
lg("lo - redirect %v", r.URL.String())
http.Redirect(w, r, r.URL.String(), http.StatusFound)
}
/*
To distinguish between posted and getted value,
we check the "post-only" slice of values first.
If nothing's there, but FormValue *has* a value,
then it was "getted", otherwise "posted"
*/
rURL := ""
urlAs := ""
err := r.ParseForm()
lg(err)
if r.PostFormValue(routes.URLParamKey) != "" {
urlAs += "url posted "
rURL = r.PostFormValue(routes.URLParamKey)
}
if r.FormValue(routes.URLParamKey) != "" {
if rURL == "" {
urlAs += "url getted "
rURL = r.FormValue(routes.URLParamKey)
}
}
// lg("received %v: %q", urlAs, rURL)
if len(rURL) == 0 {
tplAdder, tplExec := tplx.FuncTplBuilder(w, r)
tplAdder("n_html_title", "Fetch some http data", nil)
m := map[string]string{
"protocol": "https",
"host": r.Host, // not fetch.HostFromReq(r)
"path": routes.ProxifyURI,
"name": routes.URLParamKey,
"val": "google.com",
}
if util_appengine.IsLocalEnviron() {
m["protocol"] = "http"
}
tplAdder("n_cont_0", c_formFetchUrl, m)
tplExec(w, r)
} else {
r.Header.Set("X-Custom-Header-Counter", "nocounter")
bts, inf, err := fetch.UrlGetter(r, fetch.Options{URL: rURL})
lg(err)
tp := mime.TypeByExtension(path.Ext(inf.URL.Path))
if false {
ext := path.Ext(rURL)
ext = strings.ToLower(ext)
tp = mime.TypeByExtension(ext)
}
w.Header().Set("Content-Type", tp)
// w.Header().Set("Content-type", "text/html; charset=latin-1")
if r.FormValue("dbg") != "" {
w.Header().Set("Content-type", "text/html; charset=utf-8")
fmt.Fprintf(w, "%s<br>\n %s<br>\n %v", inf.URL.Path, tp, inf.URL.String())
return
}
opts := domclean2.CleaningOptions{Proxify: true}
opts.Beautify = true // "<a> Linktext without trailing space"
opts.RemoteHost = fetch.HostFromStringUrl(rURL)
// opts.ProxyHost = routes.AppHost()
opts.ProxyHost = fetch.HostFromReq(r)
if !util_appengine.IsLocalEnviron() {
opts.ProxyHost = fetch.HostFromReq(r)
}
doc, err := domclean2.DomClean(bts, opts)
var bufRend bytes.Buffer
err = html.Render(&bufRend, doc)
lg(err)
w.Write(bufRend.Bytes())
}
}