func searchPut(w http.ResponseWriter, r *http.Request) {

	id := "PA6-5001"
	user := &User{
		Customer:  "Carl Corral",
		Comment:   "I am <em>riled up</em> text",
		Visits:    1,
		LastVisit: time.Now(),
		Birthday:  time.Date(1968, time.May, 19, 0, 0, 0, 0, time.UTC),
	}

	c := appengine.NewContext(r)

	index, err := search.Open("users")
	util_err.Err_log(err)

	ret_id, err := index.Put(c, id, user)
	util_err.Err_log(err)

	fmt.Fprint(w, "OK, saved "+ret_id+"\n\n")

	var u2 User
	err = index.Get(c, ret_id, &u2)
	util_err.Err_log(err)
	fmt.Fprint(w, "Retrieved document: ", u2)

}
func readBothNamespaces(w http.ResponseWriter, r *http.Request) {

	var c1, c2 int64
	var s1, s2 string
	var err error
	var reset bool = false

	p := r.FormValue("reset")
	if p != "" {
		reset = true
	}

	c := appengine.NewContext(r)
	c1, err = agnosticReadReset(c, reset)
	util_err.Err_log(err)

	{
		c, err = appengine.Namespace(c, altNamespace)
		util_err.Err_log(err)
		c2, err = agnosticReadReset(c, reset)
		util_err.Err_log(err)
	}

	s1 = fmt.Sprintf("%v", c1)
	s2 = fmt.Sprintf("%v", c2)

	io.WriteString(w, "|"+s1+"|    |"+s2+"|")
	if reset {
		io.WriteString(w, "     and reset")
	}

}
// unused
// probably  efficient enough just to call
// var bEnc []byte = []byte(sEnc)
func StringToVByte(s string) (*bytes.Buffer, *bytes.Buffer) {

	bMsg := new(bytes.Buffer)

	bDst := new(bytes.Buffer)

	const chunksize = 20
	lb := make([]byte, chunksize) // loop buffer
	rdr := strings.NewReader(s)
	for {
		n1, err := rdr.Read(lb)
		if err == io.EOF {
			break
		}
		util_err.Err_log(err)
		if n1 < 1 {
			break
		}

		independentCopy := make([]byte, n1)
		copy(independentCopy, lb)
		n2, err := bDst.Write(independentCopy)
		util_err.Err_log(err)

		bMsg.WriteString(fmt.Sprintln("reading", n1, "bytes - writing", n2, "bytes: \n"))
		bMsg.WriteString(fmt.Sprint(" --", string(independentCopy), "--<br>\n"))
	}

	return bDst, bMsg

}
func incrementBothNamespaces(w http.ResponseWriter, r *http.Request) {

	c := appengine.NewContext(r)
	err := agnosticIncrement(c)
	util_err.Err_log(err)

	{
		c, err := appengine.Namespace(c, altNamespace)
		util_err.Err_log(err)
		err = agnosticIncrement(c)
		util_err.Err_log(err)
	}

	s := `counters updates für ns=''  and ns='ns01'.` + "\n"
	io.WriteString(w, s)
	readBothNamespaces(w, r)

}
func Base64_str_to_img(base64_img string) (img image.Image, format string) {

	pos := base64HeaderPosition(base64_img)

	reader := base64.NewDecoder(base64.StdEncoding, strings.NewReader(base64_img[pos:]))
	img, format, err := image.Decode(reader)
	util_err.Err_log(err)
	return
}
Exemple #6
0
// based on bytes.Buffer and Writing into it
func VVByte_to_string(m [][]byte) (*bytes.Buffer, *bytes.Buffer) {

	bRet := new(bytes.Buffer)
	bMsg := new(bytes.Buffer)

	//for i,v := range m {
	for i := 0; i < len(m); i++ {
		n, err := bRet.Write(m[i])
		util_err.Err_log(err)
		bMsg.WriteString(" lp" + util.Itos(i) + ": writing " + util.Itos(n) + " bytes: \n")
	}
	return bRet, bMsg
}
// BufGet - buffered get - fetches value from memory, or from memcache or from the datastore
//   todo: Paramter to reach "through" to datastore - without the buffer layers
func BufGet(w http.ResponseWriter, r *http.Request, mkk string) (WrapBlob, error) {

	c := appengine.NewContext(r)
	wb1 := new(WrapBlob)

	// first check instance memory
	wb1, ok := memoryInstanceStore[mkk]
	if ok {
		c.Infof("received %q from static instance memory", mkk)
		//util_err.StackTrace(6)
		return *wb1, nil
	}

	// secondly check memcache
	ok = McacheGet(c, mkk, wb1)
	if ok && wb1 != nil && wb1.Name != "" {
		// we could replenish memcache TTL here - instead we do that below
		c.Infof("retrieved from memcache - combi_key %v", mkk)
		memoryInstanceStore[mkk] = wb1
		return *wb1, nil
	}

	// third: retrieve from datastore
	var wb2 = WrapBlob{}
	vk := strings.Split(mkk, "__")
	if len(vk) != 2 {
		return WrapBlob{}, fmt.Errorf("key must have one '__' delimiter; %q, size %v", mkk, len(vk))
	}
	t := vk[0]
	skey := vk[1]

	key := datastore.NewKey(c, t, skey, 0, nil)
	err := datastore.Get(c, key, &wb2)
	util_err.Err_log(err)
	// missing entity and a present entity will both work.
	if err != nil && err != datastore.ErrNoSuchEntity {
		return wb2, err
	}
	McacheSet(c, mkk, wb2)
	memoryInstanceStore[mkk] = &wb2

	c.Infof("retrieved from ds - re-inserted into memcache + instance RAM - combi_key %v", mkk)

	return wb2, nil
}
// convert image to string, prepend mime header
//		inspiration_1 http://stackoverflow.com/questions/22945486/golang-converting-image-image-to-byte
//		inspiration_2 https://github.com/polds/imgbase64/blob/master/images.go
//
// mime type is always image/png
// because we only accept *image.RGBA and use image/png Encoder
func Rgba_img_to_base64_str(img *image.RGBA) string {

	// img to bytes
	buf := new(bytes.Buffer)
	err := png.Encode(buf, img)
	util_err.Err_log(err)
	imgBytes := buf.Bytes()

	// binary bytes to base64 bytes
	e64 := base64.StdEncoding
	maxEncLen := e64.EncodedLen(len(imgBytes))
	imgEnc := make([]byte, maxEncLen)
	e64.Encode(imgEnc, imgBytes)

	// base64 bytes to string
	mimeType := "image/png"
	return fmt.Sprintf("data:%s;base64,%s", mimeType, imgEnc)

}
func searchRetrieve(w http.ResponseWriter, r *http.Request) {

	c := appengine.NewContext(r)
	index, err := search.Open("users")
	util_err.Err_log(err)

	for t := index.Search(c, "Comment:riled", nil); ; {
		var res User
		id, err := t.Next(&res)
		fmt.Fprintf(w, "\n-- ")
		if err == search.Done {
			break
		}
		if err != nil {
			fmt.Fprintf(w, "Search error: %v\n", err)
			break
		}
		fmt.Fprintf(w, "%s -> %#v\n", id, res)
	}
}
Exemple #10
0
func String_to_VVByte(base64_img string) ([][]byte, *bytes.Buffer) {

	bMsg := new(bytes.Buffer)

	const chunksize = 400 //

	var size_o int
	if len(base64_img)%chunksize == 0 {
		size_o = len(base64_img) / chunksize
	} else {
		size_o = len(base64_img)/chunksize + 1
	}

	VVByte := make([][]byte, size_o)

	cntr := -1
	b := make([]byte, chunksize)
	rdr := strings.NewReader(base64_img)
	for {
		cntr++
		n, err := rdr.Read(b)
		if err == io.EOF {
			break
		}
		util_err.Err_log(err)
		if n < 1 {
			break
		}

		indep_copy := make([]byte, n)
		copy(indep_copy, b)
		VVByte[cntr] = indep_copy

		bMsg.WriteString("reading " + util.Itos(n) + " bytes:\n")
		//bMsg.Write(  VVByte[util.Itos(cntr)]  )
	}

	return VVByte, bMsg

}
func queuePush(w http.ResponseWriter, r *http.Request) {

	c := appengine.NewContext(r)

	m := map[string][]string{"counter_name": []string{nscStringKey}}
	t := taskqueue.NewPOSTTask("/_ah/namespaced-counters/queue-pop", m)

	taskqueue.Add(c, t, "")

	c, err := appengine.Namespace(c, altNamespace)
	util_err.Err_log(err)
	taskqueue.Add(c, t, "")

	io.WriteString(w, "tasks enqueued\n")

	io.WriteString(w, "\ncounter values now: \n")
	readBothNamespaces(w, r)

	io.WriteString(w, "\n\n...sleeping... \n")
	time.Sleep(time.Duration(400) * time.Millisecond)
	readBothNamespaces(w, r)

}
// saving some data by kind and key
//   without ancestor key
func saveURLNoAnc(w http.ResponseWriter, r *http.Request) {

	c := appengine.NewContext(r)

	k := ds.NewKey(c, kindUrl, keyUrl, 0, nil)
	e := new(LastURL)
	err := ds.Get(c, k, e)
	if err == ds.ErrNoSuchEntity {
		util_err.Err_log(err)
	} else {
		util_err.Err_http(w, r, err, false)
	}

	old := e.Value
	e.Value = r.URL.Path + r.URL.RawQuery

	_, err = ds.Put(c, k, e)
	util_err.Err_http(w, r, err, false)

	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	w.Write([]byte("old=" + old + "\n"))
	w.Write([]byte("new=" + e.Value + "\n"))

}
Exemple #13
0
func renameOrDelete(c appengine.Context, w http.ResponseWriter, r *http.Request) {

	b1 := new(bytes.Buffer)
	s1 := ""

	defer func() {
		w.Header().Set("Content-type", "text/html; charset=utf-8")
		w.Write(b1.Bytes())
	}()

	// c := appengine.NewContext(r)

	bk := r.FormValue("blobkey")
	if bk == "" {
		b1.WriteString("No blob key given<br>")
		return
	} else {
		s1 = fmt.Sprintf("Blob key given %q<br>", bk)
		b1.WriteString(s1)
	}

	dsKey := datastore.NewKey(c, "__BlobInfo__", bk, 0, nil)

	q := datastore.NewQuery("__BlobInfo__").Filter("__key__=", dsKey)

	var bi BlobInfo
	var found bool

	for t := q.Run(c); ; {
		_, err := t.Next(&bi)
		if err == datastore.Done {
			c.Infof("   No Results (any more), blob-rename-delete %v", err)
			break
		}
		// other err
		if err != nil {
			util_err.Err_log(err)
			return
		}
		found = true
		break
	}

	if found {
		ac := r.FormValue("action")
		if ac == "delete" {
			b1.WriteString("deletion  ")

			// first the binary data
			keyBlob, err := blobstore.BlobKeyForFile(c, bi.Filename)
			util_err.Err_log(err)

			if err != nil {
				b1.WriteString(fmt.Sprintf(" ... failed (1) %v", err))
			} else {
				err = blobstore.Delete(c, keyBlob)
				util_err.Err_log(err)
				if err != nil {
					b1.WriteString(fmt.Sprintf(" ... failed (2) %v<br>", err))
				} else {
					// now the datastore record
					err = datastore.Delete(c, dsKey)
					util_err.Err_log(err)
					if err != nil {
						b1.WriteString(fmt.Sprintf(" ... failed (3) %v<br>%#v<br>", err, dsKey))
					} else {
						b1.WriteString(" ... succeeded<br>")
					}

				}
			}
		}

		if ac == "rename" {
			b1.WriteString("renaming ")

			nfn := r.FormValue("filename")
			if nfn == "" || len(nfn) < 4 {
				b1.WriteString(" ... failed - at LEAST 4 chars required<br>")
				return
			}
			nfn = strings.ToLower(nfn)
			bi.Filename = nfn
			_, err := datastore.Put(c, dsKey, &bi)
			util_err.Err_log(err)
			if err != nil {
				b1.WriteString(fmt.Sprintf(" ... failed. %v", err))
			} else {
				b1.WriteString(" ... succeeded<br>")
			}
		}

	} else {
		b1.WriteString("no blob found for given blobkey<br>")
	}
	b1.WriteString("<a href='/blob2'>Back to list</a><br>")

}
func queuePop(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	err := agnosticIncrement(c)
	c.Infof("qp")
	util_err.Err_log(err)
}
// McacheSet is a generic memcache saving function.
// It takes scalars as well as structs.
//
// Integers and strings   are put into the memcache Value []byte
// structs			      are put into the memcache *Object* - using memcache.JSON
// Todo: types WrapString and WrapInt should be handled like string/int
//
// Scalars are tentatively saved using the CAS (compare and save) methods
func McacheSet(c appengine.Context, skey string, str_int_struct interface{}) {

	var err error
	var val string

	tMold := reflect.TypeOf(str_int_struct)
	stMold := tMold.Name()                     // strangely this is empty
	stMold = fmt.Sprintf("%T", str_int_struct) // unlike this

	if stMold != "int" &&
		stMold != "string" &&
		stMold != "dsu.WrapInt" &&
		stMold != "dsu.WrapString" {
		// struct - save it with JSON encoder
		n := tMold.NumField()
		_ = n
		miPut := &memcache.Item{
			Key:        skey,
			Value:      []byte(tMold.Name()), // sadly - value is ignored
			Object:     &str_int_struct,
			Expiration: 3600 * time.Second,
		}
		memcache.JSON.Set(c, miPut)
		c.Infof("mcache set obj key %v[%s]  - err %v", skey, stMold, err)

	} else {
		// scalar value - save it
		switch chamaeleon := str_int_struct.(type) {
		default:
			panic(fmt.Sprintf("only string or int - instead: -%T", str_int_struct))
		case nil:
			val = ""
		case WrapString:
			val = chamaeleon.S
		case string:
			val = chamaeleon
		case int:
			val = util.Itos(chamaeleon)
		case WrapInt:
			val = util.Itos(chamaeleon.I)
		}

		/*
			This is a Compare and Set (CAS) implementation of "set".
			It implements optimistic locking.

			We fetch the item first, then modify it, then put it back.
			We rely on the hidden "casID" of the memcache item,
				to detect intermittent changes by competitors.

			Biggest downside is the additional roundtrip for the fetch.
			Second downside: We should implement a retry after failure.
				Instead I resorted to a simple "SET"

			Upside: Prevention of race conditions.
				But race conditions only matter if newval = f(oldval)
				Otherwise last one wins should apply anyway.

		*/

		maxTries := 3

		miCas, eget := memcache.Get(c, skey) // compare and swap

		for i := 0; i <= maxTries; i++ {

			if i == maxTries {
				panic(fmt.Sprintf("memcache set CAS failed after %v attempts", maxTries))
			}

			var eput error
			var putMode = ""
			if eget != memcache.ErrCacheMiss {
				putMode = "CAS"
				miCas.Value = []byte(val)
				eput = memcache.CompareAndSwap(c, miCas)
			} else {
				putMode = "ADD"
				miCas := &memcache.Item{
					Key:   skey,
					Value: []byte(val),
				}
				eput = memcache.Add(c, miCas)
			}

			if eput == memcache.ErrCASConflict {
				c.Errorf("\t memcache CAS  FAILED - concurrent update?")
				// we brutally fallback to set():
				miCas := &memcache.Item{
					Key:   skey,
					Value: []byte(val),
				}
				eset := memcache.Set(c, miCas)
				util_err.Err_log(eset)
				time.Sleep(10 * time.Millisecond)
				continue
			}
			if eput == memcache.ErrNotStored {
				c.Errorf("\t memcache save FAILED - no idea why it would")
				time.Sleep(10 * time.Millisecond)
				continue
			}

			c.Infof("mcache set scalar %v[%T]=%v - mode %v - eget/eput: %v/%v",
				skey, str_int_struct, val, putMode, eget, eput)
			break
		}

	}

}