func Get(w http.ResponseWriter, r *http.Request, m map[string]interface{}) *Instance {
c := appengine.NewContext(r)
startFunc := time.Now()
if !ii.LastUpdated.IsZero() {
age := startFunc.Sub(ii.LastUpdated)
if age < 200*time.Millisecond {
c.Infof("instance info update too recently: %v, skipping.\n", age)
return ii
}
if age < 1*time.Hour {
if len(ii.Hostname) > 2 {
return ii
}
}
c.Infof("instance info update too old: %v, recomputing.\n", age)
}
ii.ModuleName = appengine.ModuleName(c)
ii.InstanceID = appengine.InstanceID()
ii.VersionFull = appengine.VersionID(c)
majorMinor := strings.Split(ii.VersionFull, ".")
if len(majorMinor) != 2 {
panic("we need a version string of format X.Y")
}
ii.VersionMajor = majorMinor[0]
ii.VersionMinor = majorMinor[1]
var err = errors.New("dummy creation error message")
ii.NumInstances, err = module.NumInstances(c, ii.ModuleName, ii.VersionFull)
if err != nil {
// this never works with version full
// we do not log this - but try version major
err = nil
if !util_appengine.IsLocalEnviron() {
ii.NumInstances, err = module.NumInstances(c, ii.ModuleName, ii.VersionMajor)
util_err.Err_http(w, r, err, true, "get num instances works only live and without autoscale")
}
}
// in auto scaling, google reports "zero" - which can not be true
// we assume at least 1
if ii.NumInstances == 0 {
ii.NumInstances = 1
}
// http://[0-2].1.default.libertarian-islands.appspot.com/instance-info
ii.Hostname, err = appengine.ModuleHostname(c, ii.ModuleName,
ii.VersionMajor, "")
util_err.Err_http(w, r, err, false)
if !util_appengine.IsLocalEnviron() {
ii.HostnameInst0, err = appengine.ModuleHostname(c, ii.ModuleName,
ii.VersionMajor, "0")
if err != nil && (err.Error() == autoScalingErr1 || err.Error() == autoScalingErr2) {
c.Infof("inst 0: " + autoScalingErrMsg)
err = nil
}
util_err.Err_http(w, r, err, true)
ii.HostnameInst1, err = appengine.ModuleHostname(c, ii.ModuleName,
ii.VersionMajor, "1")
if err != nil && (err.Error() == autoScalingErr1 || err.Error() == autoScalingErr2) {
c.Infof("inst 1: " + autoScalingErrMsg)
err = nil
}
util_err.Err_http(w, r, err, true)
ii.HostnameMod02, err = appengine.ModuleHostname(c, "mod02", "", "")
util_err.Err_http(w, r, err, true)
}
ii.LastUpdated = time.Now()
c.Infof("collectInfo() completed, %v.%v.%v.%v, took %v",
util.Ellipsoider(ii.InstanceID, 4), ii.VersionMajor, ii.ModuleName,
ii.Hostname,
ii.LastUpdated.Sub(startFunc))
return ii
}
func parseFurther(w http.ResponseWriter, r *http.Request, saveImages bool) {
c := appengine.NewContext(r)
b := new(bytes.Buffer)
defer func() {
w.Header().Set("Content-type", "text/plain; charset=utf-8")
w.Write(b.Bytes())
}()
// Get the item from the memcache
wb1 := new(dsu.WrapBlob)
ok := dsu.McacheGet(c, keyLatest, wb1)
util_err.Err_http(w, r, ok, true)
if ok {
b.WriteString(sp("name %v\n", wb1.Name))
b.WriteString(sp("S (boundary): %q\n", wb1.S))
// dumps the entire body
// b.WriteString(sp("B: %v\n", string(wb1.VByte)))
// instead we split it by multipart mime
vb := bytes.Split(wb1.VByte, []byte("--"+wb1.S))
for i, v := range vb {
h := "" // header
fn := "" // filename
s := string(v)
s = strings.Trim(s, "\r \n")
ctype := ""
b.WriteString(sp("\n___________mime boundary index %v___________\n", i))
if strings.HasPrefix(s, "Content-Type: image/png;") ||
strings.HasPrefix(s, "Content-Type: image/jpeg;") {
if start := strings.Index(s, sepHeaderContent); start > 0 {
h = s[:start]
vh := strings.Split(h, "\r\n")
for _, v := range vh {
v := strings.TrimSpace(v)
// b.WriteString("\t\t" + v + "\n")
if strings.HasPrefix(v, "name=") {
vv := strings.Split(v, "=")
fn = util.LowerCasedUnderscored(vv[1])
}
}
s = s[start+len(sepHeaderContent):]
if posSemicol := strings.Index(h, ";"); posSemicol > 0 {
ctype = h[0:posSemicol]
}
}
}
if ctype == "" {
b.WriteString("unparseable: " + util.Ellipsoider(s, 400))
} else {
b.WriteString(sp("\n\tctype=%v\n\t------------", ctype))
if fn != "" {
b.WriteString(sp("\n\tfilename=%v\n\t------------", fn))
}
if saveImages {
rE := resEntry{}
rE.when = util.TimeMarker()
rE.contentType = ctype
rE.fn = fn
rE.b64Img = &s
Images[reservoirRevolver%reservoirSize] = rE
reservoirRevolver++
c.Infof("Put image into reservoir %v %v", fn, ctype)
}
}
}
}
}
func cleanseHtml(r io.Reader) (*bytes.Buffer, error) {
skip := map[string]string{
"script": "skip",
"noscript": "skip",
"link": "skip",
"meta": "skip",
"iframe": "skip",
}
b := new(bytes.Buffer)
d := html.NewTokenizer(r)
cntrErr := 0
cntrTkn := 0
fuckOff := false
for {
tokenType := d.Next()
cntrTkn++
if tokenType == html.ErrorToken {
cntrErr++
if cntrErr > 5 {
return b, errors.New(spf("error loop at pos %v", cntrTkn))
}
continue
}
token := d.Token()
s2 := strings.TrimSpace(string(token.Data))
attr := getAttr(token.Attr)
cntrErr = 0
switch tokenType {
case html.StartTagToken:
if _, ok := skip[s2]; ok {
fuckOff = true
} else {
s2 = "\n<" + s2 + attr + ">"
}
case html.EndTagToken: // </tag>
if _, ok := skip[s2]; ok {
fuckOff = false
s2 = ""
} else {
// s2 = "</" + s2 + ">"
s2 = "\n</" + s2 + ">\n"
}
case html.SelfClosingTagToken:
if _, ok := skip[s2]; ok {
s2 = ""
} else {
s2 = "\n<" + s2 + attr + "/>\n"
}
case html.DoctypeToken:
s2 = "<!DOCTYPE " + s2 + `><meta content="text/html; charset=utf-8" http-equiv="Content-Type"/>`
case html.TextToken:
// nothing
case html.CommentToken:
s2 = ""
default:
// nothing
}
if !fuckOff {
b.WriteString(s2)
} else {
if s2 != "" {
s2 = strings.Replace(s2, "\n", "", -1)
s2 = util.Ellipsoider(s2, 30)
pf("skipped %v \n", s2)
}
}
}
return b, nil
}
func guestViewCursor(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {
c := appengine.NewContext(r)
q := ds.NewQuery(gbp.DSKindGBEntry)
q.Order("-Date")
b1 := new(bytes.Buffer)
cur_start, err := memcache.Get(c, "greeting_cursor")
if err == nil {
str_curs := string(cur_start.Value)
if len(cur_start.Value) > 0 {
cursor, err := ds.DecodeCursor(str_curs) // inverse is string()
util_err.Err_http(w, r, err, false)
if err == nil {
b1.WriteString("found cursor from memcache -" + util.Ellipsoider(str_curs, 10) + "-<br>\n")
q = q.Start(cursor)
}
}
}
iter := q.Run(c)
var cntr int = 0
for {
var g gbp.GbEntryRetr
cntr++
if cntr > 2 {
b1.WriteString(" batch complete -" + string(cntr) + "-<br>\n")
break
}
_, err := iter.Next(&g)
if err == ds.Done {
b1.WriteString("scan complete -" + string(cntr) + "-<br>\n")
break
}
if fmt.Sprintf("%T", err) == fmt.Sprintf("%T", new(ds.ErrFieldMismatch)) {
err = nil // ignore this one - it's caused by our deliberate differences between gbsaveEntry and gbEntrieRetr
}
if err != nil {
b1.WriteString("error fetching next: " + err.Error() + "<br>\n")
break
}
b1.WriteString(" - " + g.String())
}
// Get updated cursor and store it for next time.
if cur_end, err := iter.Cursor(); err == nil {
str_c_end := cur_end.String() // inverse is decode()
val := []byte(str_c_end)
mi_save := &memcache.Item{
Key: "greeting_cursor",
Value: val,
Expiration: 60 * time.Second,
}
if err := memcache.Set(c, mi_save); err != nil {
b1.WriteString("error adding memcache item " + err.Error() + "<br>\n")
} else {
b1.WriteString("wrote cursor to memcache -" + util.Ellipsoider(str_c_end, 10) + "-<br>\n")
}
} else {
b1.WriteString("could not retrieve cursor_end " + err.Error() + "<br>\n")
}
w.Header().Set("Content-Type", "text/html")
w.Write(b1.Bytes())
w.Write([]byte("<br>----<br>"))
}
// src http://golang-examples.tumblr.com/page/2
func decomposeHtml(r io.Reader) {
// type Token struct {
// Type TokenType
// DataAtom atom.Atom
// Data string
// Attr []Attribute
// }
// type Attribute struct {
// Namespace, Key, Val string
// }
skip := map[string]string{
"meta": "skip",
"html": "skip",
"head": "skip",
"title": "skip",
"body": "skip",
"link": "skip",
"script": "skip",
"noscript": "skip",
"----------": "skip",
"iframe": "skip",
"nav": "skip",
"form": "skip",
}
histogram := map[string]interface{}{}
d := html.NewTokenizer(r)
cntrErr := 0
cntrTkn := 0
for {
tokenType := d.Next()
cntrTkn++
if tokenType == html.ErrorToken {
pf("#%v err ", cntrTkn)
cntrErr++
if cntrErr > 5 {
break
}
continue
}
token := d.Token()
cntrErr = 0
s1 := strings.TrimSpace(spf(" %#v", token))
s2 := strings.TrimSpace(string(token.Data))
s3 := string(token.DataAtom)
_, _, _ = s1, s2, s3
switch tokenType {
case html.StartTagToken, html.SelfClosingTagToken:
if _, ok := skip[s2]; !ok {
pf("\n%v ", s2)
if _, ok := histogram[s2]; !ok {
histogram[s2] = 1
} else {
val := histogram[s2].(int)
histogram[s2] = val + 1
}
}
case html.TextToken:
if s2 != "" && len(s2) > 1 && !strings.HasPrefix(s2, `//`) {
s2 = strings.Replace(s2, "\n", "", -1)
pf("\t%v", util.Ellipsoider(s2, 22))
}
case html.EndTagToken: // </tag>
// pf("/%v ", s2)
case html.CommentToken:
// pf("comment ")
case html.DoctypeToken:
default:
pf("default case %v\n", s1)
}
}
hSort := util.StringKeysToSortedArray(histogram)
pf("\n\n")
for _, v := range hSort {
pf("%10s %4v\n", v, histogram[v])
}
}