Example #1
0
func Get(w http.ResponseWriter, r *http.Request, m map[string]interface{}) *Instance {

	c := appengine.NewContext(r)

	startFunc := time.Now()

	if !ii.LastUpdated.IsZero() {

		age := startFunc.Sub(ii.LastUpdated)

		if age < 200*time.Millisecond {
			c.Infof("instance info update too recently: %v, skipping.\n", age)
			return ii
		}

		if age < 1*time.Hour {
			if len(ii.Hostname) > 2 {
				return ii
			}

		}

		c.Infof("instance info update too old: %v, recomputing.\n", age)
	}

	ii.ModuleName = appengine.ModuleName(c)
	ii.InstanceID = appengine.InstanceID()
	ii.VersionFull = appengine.VersionID(c)

	majorMinor := strings.Split(ii.VersionFull, ".")
	if len(majorMinor) != 2 {
		panic("we need a version string of format X.Y")
	}

	ii.VersionMajor = majorMinor[0]
	ii.VersionMinor = majorMinor[1]

	var err = errors.New("dummy creation error message")

	ii.NumInstances, err = module.NumInstances(c, ii.ModuleName, ii.VersionFull)
	if err != nil {
		// this never works with version full
		// we do not log this - but try version major
		err = nil

		if !util_appengine.IsLocalEnviron() {
			ii.NumInstances, err = module.NumInstances(c, ii.ModuleName, ii.VersionMajor)
			util_err.Err_http(w, r, err, true, "get num instances works only live and without autoscale")
		}

	}

	// in auto scaling, google reports "zero" - which can not be true
	// we assume at least 1
	if ii.NumInstances == 0 {
		ii.NumInstances = 1
	}

	// http://[0-2].1.default.libertarian-islands.appspot.com/instance-info

	ii.Hostname, err = appengine.ModuleHostname(c, ii.ModuleName,
		ii.VersionMajor, "")
	util_err.Err_http(w, r, err, false)

	if !util_appengine.IsLocalEnviron() {
		ii.HostnameInst0, err = appengine.ModuleHostname(c, ii.ModuleName,
			ii.VersionMajor, "0")
		if err != nil && (err.Error() == autoScalingErr1 || err.Error() == autoScalingErr2) {
			c.Infof("inst 0: " + autoScalingErrMsg)
			err = nil
		}
		util_err.Err_http(w, r, err, true)

		ii.HostnameInst1, err = appengine.ModuleHostname(c, ii.ModuleName,
			ii.VersionMajor, "1")
		if err != nil && (err.Error() == autoScalingErr1 || err.Error() == autoScalingErr2) {
			c.Infof("inst 1: " + autoScalingErrMsg)
			err = nil
		}
		util_err.Err_http(w, r, err, true)

		ii.HostnameMod02, err = appengine.ModuleHostname(c, "mod02", "", "")
		util_err.Err_http(w, r, err, true)

	}

	ii.LastUpdated = time.Now()

	c.Infof("collectInfo() completed, %v.%v.%v.%v, took %v",
		util.Ellipsoider(ii.InstanceID, 4), ii.VersionMajor, ii.ModuleName,
		ii.Hostname,
		ii.LastUpdated.Sub(startFunc))

	return ii
}
Example #2
0
func parseFurther(w http.ResponseWriter, r *http.Request, saveImages bool) {

	c := appengine.NewContext(r)

	b := new(bytes.Buffer)
	defer func() {
		w.Header().Set("Content-type", "text/plain; charset=utf-8")
		w.Write(b.Bytes())
	}()

	// Get the item from the memcache
	wb1 := new(dsu.WrapBlob)
	ok := dsu.McacheGet(c, keyLatest, wb1)
	util_err.Err_http(w, r, ok, true)

	if ok {
		b.WriteString(sp("name %v\n", wb1.Name))
		b.WriteString(sp("S (boundary): %q\n", wb1.S))

		// dumps the entire body
		// b.WriteString(sp("B: %v\n", string(wb1.VByte)))

		// instead we split it by multipart mime
		vb := bytes.Split(wb1.VByte, []byte("--"+wb1.S))
		for i, v := range vb {
			h := ""  // header
			fn := "" // filename
			s := string(v)
			s = strings.Trim(s, "\r \n")
			ctype := ""

			b.WriteString(sp("\n___________mime boundary index %v___________\n", i))
			if strings.HasPrefix(s, "Content-Type: image/png;") ||
				strings.HasPrefix(s, "Content-Type: image/jpeg;") {

				if start := strings.Index(s, sepHeaderContent); start > 0 {
					h = s[:start]
					vh := strings.Split(h, "\r\n")
					for _, v := range vh {
						v := strings.TrimSpace(v)
						// b.WriteString("\t\t" + v + "\n")
						if strings.HasPrefix(v, "name=") {
							vv := strings.Split(v, "=")
							fn = util.LowerCasedUnderscored(vv[1])
						}
					}
					s = s[start+len(sepHeaderContent):]
					if posSemicol := strings.Index(h, ";"); posSemicol > 0 {
						ctype = h[0:posSemicol]
					}
				}
			}

			if ctype == "" {
				b.WriteString("unparseable: " + util.Ellipsoider(s, 400))
			} else {
				b.WriteString(sp("\n\tctype=%v\n\t------------", ctype))
				if fn != "" {
					b.WriteString(sp("\n\tfilename=%v\n\t------------", fn))
				}
				if saveImages {
					rE := resEntry{}
					rE.when = util.TimeMarker()
					rE.contentType = ctype
					rE.fn = fn
					rE.b64Img = &s
					Images[reservoirRevolver%reservoirSize] = rE
					reservoirRevolver++
					c.Infof("Put image into reservoir %v %v", fn, ctype)
				}
			}

		}

	}

}
Example #3
0
func cleanseHtml(r io.Reader) (*bytes.Buffer, error) {

	skip := map[string]string{
		"script":   "skip",
		"noscript": "skip",
		"link":     "skip",
		"meta":     "skip",
		"iframe":   "skip",
	}

	b := new(bytes.Buffer)

	d := html.NewTokenizer(r)
	cntrErr := 0
	cntrTkn := 0
	fuckOff := false
	for {
		tokenType := d.Next()
		cntrTkn++

		if tokenType == html.ErrorToken {
			cntrErr++
			if cntrErr > 5 {
				return b, errors.New(spf("error loop at pos %v", cntrTkn))
			}
			continue
		}

		token := d.Token()
		s2 := strings.TrimSpace(string(token.Data))
		attr := getAttr(token.Attr)

		cntrErr = 0
		switch tokenType {
		case html.StartTagToken:
			if _, ok := skip[s2]; ok {
				fuckOff = true
			} else {
				s2 = "\n<" + s2 + attr + ">"
			}
		case html.EndTagToken: // </tag>
			if _, ok := skip[s2]; ok {
				fuckOff = false
				s2 = ""
			} else {
				// s2 = "</" + s2 + ">"
				s2 = "\n</" + s2 + ">\n"
			}
		case html.SelfClosingTagToken:
			if _, ok := skip[s2]; ok {
				s2 = ""
			} else {
				s2 = "\n<" + s2 + attr + "/>\n"
			}
		case html.DoctypeToken:
			s2 = "<!DOCTYPE " + s2 + `><meta content="text/html; charset=utf-8" http-equiv="Content-Type"/>`

		case html.TextToken:
			// nothing
		case html.CommentToken:
			s2 = ""
		default:
			// nothing
		}

		if !fuckOff {
			b.WriteString(s2)
		} else {
			if s2 != "" {
				s2 = strings.Replace(s2, "\n", "", -1)
				s2 = util.Ellipsoider(s2, 30)
				pf("skipped %v \n", s2)

			}
		}
	}
	return b, nil

}
func guestViewCursor(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {

	c := appengine.NewContext(r)

	q := ds.NewQuery(gbp.DSKindGBEntry)
	q.Order("-Date")

	b1 := new(bytes.Buffer)

	cur_start, err := memcache.Get(c, "greeting_cursor")
	if err == nil {
		str_curs := string(cur_start.Value)
		if len(cur_start.Value) > 0 {
			cursor, err := ds.DecodeCursor(str_curs) //  inverse is string()
			util_err.Err_http(w, r, err, false)
			if err == nil {
				b1.WriteString("found cursor from memcache -" + util.Ellipsoider(str_curs, 10) + "-<br>\n")
				q = q.Start(cursor)
			}
		}
	}

	iter := q.Run(c)
	var cntr int = 0
	for {
		var g gbp.GbEntryRetr
		cntr++
		if cntr > 2 {
			b1.WriteString("  batch complete -" + string(cntr) + "-<br>\n")
			break

		}

		_, err := iter.Next(&g)
		if err == ds.Done {
			b1.WriteString("scan complete -" + string(cntr) + "-<br>\n")
			break
		}

		if fmt.Sprintf("%T", err) == fmt.Sprintf("%T", new(ds.ErrFieldMismatch)) {
			err = nil // ignore this one - it's caused by our deliberate differences between gbsaveEntry and gbEntrieRetr
		}

		if err != nil {
			b1.WriteString("error fetching next: " + err.Error() + "<br>\n")
			break
		}

		b1.WriteString("  - " + g.String())
	}

	// Get updated cursor and store it for next time.
	if cur_end, err := iter.Cursor(); err == nil {

		str_c_end := cur_end.String() //  inverse is decode()
		val := []byte(str_c_end)

		mi_save := &memcache.Item{
			Key:        "greeting_cursor",
			Value:      val,
			Expiration: 60 * time.Second,
		}

		if err := memcache.Set(c, mi_save); err != nil {
			b1.WriteString("error adding memcache item " + err.Error() + "<br>\n")
		} else {
			b1.WriteString("wrote cursor to memcache -" + util.Ellipsoider(str_c_end, 10) + "-<br>\n")
		}

	} else {
		b1.WriteString("could not retrieve cursor_end " + err.Error() + "<br>\n")
	}

	w.Header().Set("Content-Type", "text/html")
	w.Write(b1.Bytes())

	w.Write([]byte("<br>----<br>"))

}
Example #5
0
// src http://golang-examples.tumblr.com/page/2
func decomposeHtml(r io.Reader) {

	// type Token struct {
	//     Type     TokenType
	//     DataAtom atom.Atom
	//     Data     string
	//     Attr     []Attribute
	// }
	// type Attribute struct {
	//     Namespace, Key, Val string
	// }

	skip := map[string]string{
		"meta":       "skip",
		"html":       "skip",
		"head":       "skip",
		"title":      "skip",
		"body":       "skip",
		"link":       "skip",
		"script":     "skip",
		"noscript":   "skip",
		"----------": "skip",
		"iframe":     "skip",
		"nav":        "skip",
		"form":       "skip",
	}
	histogram := map[string]interface{}{}

	d := html.NewTokenizer(r)
	cntrErr := 0
	cntrTkn := 0
	for {
		tokenType := d.Next()
		cntrTkn++

		if tokenType == html.ErrorToken {
			pf("#%v err ", cntrTkn)
			cntrErr++
			if cntrErr > 5 {
				break
			}
			continue
		}

		token := d.Token()
		cntrErr = 0
		s1 := strings.TrimSpace(spf(" %#v", token))
		s2 := strings.TrimSpace(string(token.Data))
		s3 := string(token.DataAtom)
		_, _, _ = s1, s2, s3

		switch tokenType {
		case html.StartTagToken, html.SelfClosingTagToken:
			if _, ok := skip[s2]; !ok {
				pf("\n%v ", s2)
				if _, ok := histogram[s2]; !ok {
					histogram[s2] = 1
				} else {
					val := histogram[s2].(int)
					histogram[s2] = val + 1
				}
			}
		case html.TextToken:
			if s2 != "" && len(s2) > 1 && !strings.HasPrefix(s2, `//`) {
				s2 = strings.Replace(s2, "\n", "", -1)
				pf("\t%v", util.Ellipsoider(s2, 22))
			}
		case html.EndTagToken: // </tag>
			// pf("/%v ", s2)
		case html.CommentToken:
			// pf("comment ")
		case html.DoctypeToken:

		default:
			pf("default case %v\n", s1)
		}
	}

	hSort := util.StringKeysToSortedArray(histogram)

	pf("\n\n")
	for _, v := range hSort {
		pf("%10s %4v\n", v, histogram[v])
	}

}