// Increment increments the named counter. func Increment(c context.Context, valName string) error { // Get counter config. shardsTotal := dsu.WrapInt{} dsu.McacheGet(c, mcKeyShardsTotal(valName), &shardsTotal) if shardsTotal.I < 1 { ckey := datastore.NewKey(c, dsKindNumShards, mcKeyShardsTotal(valName), 0, nil) errTx := datastore.RunInTransaction(c, func(c context.Context) error { err := datastore.Get(c, ckey, &shardsTotal) if err == datastore.ErrNoSuchEntity { shardsTotal.I = defaultNumShards _, err = datastore.Put(c, ckey, &shardsTotal) } return err }, nil) if errTx != nil { return errTx } dsu.McacheSet(c, mcKeyShardsTotal(valName), dsu.WrapInt{shardsTotal.I}) } // pick random counter and increment it errTx := datastore.RunInTransaction(c, func(c context.Context) error { shardId := rand.Intn(shardsTotal.I) dsKey := datastore.NewKey(c, dsKindShard, keySingleShard(valName, shardId), 0, nil) var sd WrapShardData err := datastore.Get(c, dsKey, &sd) // A missing entity and a present entity will both work. if err != nil && err != datastore.ErrNoSuchEntity { return err } sd.Name = valName sd.ShardId = shardId sd.I++ _, err = datastore.Put(c, dsKey, &sd) if ll > 2 { aelog.Infof(c, "ds put %v %v", dsKey, sd) } return err }, nil) if errTx != nil { return errTx } memcache.Increment(c, mcKey(valName), 1, 0) // collect number of updates // per valName per instance in memory // for every interval of 10 minutes // // a batch job checks if the number of shards should be increased or decreased // and truncates this map updateSamplingFrequency[valName+util.TimeMarker()[:len("2006-01-02 15:0")]] += 1 return nil }
func saveURLWithAncestor(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { c := appengine.NewContext(r) k := ds.NewKey(c, kindUrl, "", 0, ancKey(c)) s := util.TimeMarker() ls := len(s) lc := len("09-26 17:29:25") lastURL_fictitious_1 := LastURL{"with_anc " + s[ls-lc:ls-3]} _, err := ds.Put(c, k, &lastURL_fictitious_1) loghttp.E(w, r, err, false) }
func parseFurther(w http.ResponseWriter, r *http.Request, saveImages bool) { c := appengine.NewContext(r) b := new(bytes.Buffer) defer func() { w.Header().Set("Content-type", "text/plain; charset=utf-8") w.Write(b.Bytes()) }() // Get the item from the memcache wb1 := new(dsu.WrapBlob) ok := dsu.McacheGet(c, keyLatest, wb1) loghttp.E(w, r, ok, true) if ok { b.WriteString(sp("name %v\n", wb1.Name)) b.WriteString(sp("S (boundary): %q\n", wb1.S)) // dumps the entire body // b.WriteString(sp("B: %v\n", string(wb1.VByte))) // instead we split it by multipart mime vb := bytes.Split(wb1.VByte, []byte("--"+wb1.S)) for i, v := range vb { h := "" // header fn := "" // filename s := string(v) s = strings.Trim(s, "\r \n") ctype := "" b.WriteString(sp("\n___________mime boundary index %v___________\n", i)) if strings.HasPrefix(s, "Content-Type: image/png;") || strings.HasPrefix(s, "Content-Type: image/jpeg;") { if start := strings.Index(s, sepHeaderContent); start > 0 { h = s[:start] vh := strings.Split(h, "\r\n") for _, v := range vh { v := strings.TrimSpace(v) // b.WriteString("\t\t" + v + "\n") if strings.HasPrefix(v, "name=") { vv := strings.Split(v, "=") fn = stringspb.LowerCasedUnderscored(vv[1]) } } s = s[start+len(sepHeaderContent):] if posSemicol := strings.Index(h, ";"); posSemicol > 0 { ctype = h[0:posSemicol] } } } if ctype == "" { b.WriteString("unparseable: " + stringspb.Ellipsoider(s, 400)) } else { b.WriteString(sp("\n\tctype=%v\n\t------------", ctype)) if fn != "" { b.WriteString(sp("\n\tfilename=%v\n\t------------", fn)) } if saveImages { rE := resEntry{} rE.when = util.TimeMarker() rE.contentType = ctype rE.fn = fn rE.b64Img = &s Images[reservoirRevolver%reservoirSize] = rE reservoirRevolver++ aelog.Infof(c, "Put image into reservoir %v %v", fn, ctype) } } } } }
func queryIntoDatastore(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { limitUpper := util.MonthsBack(1) limitLower := util.MonthsBack(25) var q bq.QueryRequest = bq.QueryRequest{} q.Query = ` SELECT repository_language , LEFT(repository_pushed_at,7) monthx , CEIL( count(*)/1000) Tausend FROM githubarchive:github.timeline where 1=1 AND LEFT(repository_pushed_at,7) >= '` + limitLower + `' AND LEFT(repository_pushed_at,7) <= '` + limitUpper + `' AND repository_language in ('Go','go','Golang','golang','C','Java','PHP','JavaScript','C++','Python','Ruby') AND type="PushEvent" group by monthx, repository_language order by repository_language , monthx ; ` c := appengine.NewContext(r) // The following client will be authorized by the App Engine // app's service account for the provided scopes. // "https://www.googleapis.com/auth/bigquery" // "https://www.googleapis.com/auth/devstorage.full_control" // 2015-06: instead of oauth2.NoContext we get a new type of context var ctx context.Context = appengine.NewContext(r) oauthHttpClient, err := google.DefaultClient( ctx, "https://www.googleapis.com/auth/bigquery") if err != nil { log.Fatal(err) } bigqueryService, err := bq.New(oauthHttpClient) loghttp.E(w, r, err, false) fmt.Fprint(w, "s1<br>\n") // Create a query statement and query request object // query_data = {'query':'SELECT TOP(title, 10) as title, COUNT(*) as revision_count FROM [publicdata:samples.wikipedia] WHERE wp_namespace = 0;'} // query_request = bigquery_service.jobs() // Make a call to the BigQuery API // query_response = query_request.query(projectId=PROJECT_NUMBER, body=query_data).execute() js := bq.NewJobsService(bigqueryService) jqc := js.Query("347979071940", &q) fmt.Fprint(w, "s2 "+util.TimeMarker()+" <br>\n") resp, err := jqc.Do() loghttp.E(w, r, err, false) rows := resp.Rows var vVDest [][]byte = make([][]byte, len(rows)) aelog.Errorf(c, "%#v", rows) for i0, v0 := range rows { cells := v0.F b_row := new(bytes.Buffer) b_row.WriteString(fmt.Sprintf("r%0.2d -- ", i0)) for i1, v1 := range cells { val1 := v1.V b_row.WriteString(fmt.Sprintf("c%0.2d: %v ", i1, val1)) } vVDest[i0] = []byte(b_row.Bytes()) } key_combi, _ := dsu.BufPut(c, dsu.WrapBlob{Name: "bq_res1", VVByte: vVDest}, "bq_res1") dsObj, _ := dsu.BufGet(c, key_combi) printPlaintextTable(w, r, dsObj.VVByte) fmt.Fprint(w, "s3 "+util.TimeMarker()+" <br>\n") }
/* Domain is different!!! appspotMAIL.com not appspot.com peter@[email protected] https://developers.google.com/appengine/docs/python/mail/receivingmail email-address: [email protected] is routed to /_ah/mail/[email protected] [email protected] is routed to /_ah/mail/[email protected] */ func emailReceiveAndStore(w http.ResponseWriter, r *http.Request, mx map[string]interface{}) { c := appengine.NewContext(r) defer r.Body.Close() msg, err := go_mail.ReadMessage(r.Body) loghttp.E(w, r, err, false, "could not do ReadMessage") if msg == nil { aelog.Warningf(c, "-empty msg- "+r.URL.Path) return } // see http://golang.org/pkg/net/mail/#Message b1 := new(bytes.Buffer) // for i, m1 := range msg.Header { // aelog.Infof(c,"--msg header %q : %v", i, m1) // } from := msg.Header.Get("from") + "\n" b1.WriteString("from: " + from) to := msg.Header.Get("to") + "\n" b1.WriteString("to: " + to) subject := msg.Header.Get("subject") + "\n" b1.WriteString("subject: " + subject) when, _ := msg.Header.Date() swhen := when.Format("2006-01-02 - 15:04 \n") b1.WriteString("when: " + swhen) ctype := msg.Header.Get("Content-Type") aelog.Infof(c, "content type header: %q", ctype) boundary := "" // [multipart/mixed; boundary="------------060002090509030608020402"] if strings.HasPrefix(ctype, "[multipart/mixed") || strings.HasPrefix(ctype, "multipart/mixed") { vT1 := strings.Split(ctype, ";") if len(vT1) > 1 { aelog.Infof(c, "substring 1: %q", vT1[1]) sT11 := vT1[1] sT11 = strings.TrimSpace(sT11) sT11 = strings.TrimPrefix(sT11, "boundary=") sT11 = strings.Trim(sT11, `"`) boundary = sT11 aelog.Infof(c, "substring 2: %q", boundary) } } b1.WriteString("\n\n") b1.ReadFrom(msg.Body) dsu.McacheSet(c, keyLatest, dsu.WrapBlob{Name: subject, S: boundary, VByte: b1.Bytes()}) if strings.HasPrefix(to, "foscam") { // send confirmation to sender var m map[string]string = nil m = make(map[string]string) m["sender"] = from m["subject"] = "confirmation: " + subject emailSend(w, r, m) parseFurther(w, r, true) call(w, r, mx) } else { blob := dsu.WrapBlob{Name: subject + "from " + from + "to " + to, S: boundary, VByte: b1.Bytes()} blob.VVByte, _ = conv.String_to_VVByte(b1.String()) dsu.BufPut(c, blob, "email-"+util.TimeMarker()) } }
func backend(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { w.Header().Set("Content-type", "text/html; charset=utf-8") w.WriteHeader(http.StatusOK) if ok, _, msg := login.CheckForAdminUser(r); !ok { w.Write([]byte(msg)) return } b1 := new(bytes.Buffer) b1.WriteString(tplx.ExecTplHelper(tplx.Head, map[string]interface{}{"HtmlTitle": "Backend V1"})) htmlfrag.Wb(b1, "Debug pprof", "/debug/pprof") htmlfrag.Wb(b1, "Diverse", "nobr") htmlfrag.Wb(b1, "Schreib-Methoden", "/write-methods") htmlfrag.Wb(b1, "Letzte Email", "/email-view") htmlfrag.Wb(b1, "Blob List", "/blob2") htmlfrag.Wb(b1, "fetch via proxy", routes.ProxifyURI) htmlfrag.Wb(b1, "Instance Info", "/instance-info/view") htmlfrag.Wb(b1, "Fulltext put", "/fulltext-search/put") htmlfrag.Wb(b1, "Fulltext get", "/fulltext-search/get") htmlfrag.Wb(b1, "datastore object view quoted printabe", "/dsu/show") htmlfrag.Wb(b1, "Statistics", "/_ah/stats") htmlfrag.Wb(b1, "Request Images ", "") htmlfrag.Wb(b1, "WrapBlob from Datastore", "/image/img-from-datastore?p=chart1") htmlfrag.Wb(b1, "base64 from Datastore", "/image/base64-from-datastore?p=chart1") htmlfrag.Wb(b1, "base64 from Variable", "/image/base64-from-var?p=1") htmlfrag.Wb(b1, "base64 from File", "/image/base64-from-file?p=static/pberg1.png") htmlfrag.Wb(b1, "Drawing a static chart", "/image/draw-lines-example") htmlfrag.Wb(b1, "Big Query ...", "") htmlfrag.Wb(b1, "Get real data", "/big-query/query-into-datastore") htmlfrag.Wb(b1, "Get mocked data", "/big-query/mock-data-into-datastore") htmlfrag.Wb(b1, " ... with Chart", "") htmlfrag.Wb(b1, "Process Data 1 (mock=1)", "/big-query/regroup-data-01?mock=0") htmlfrag.Wb(b1, "Process Data 2", "/big-query/regroup-data-02?f=table") htmlfrag.Wb(b1, "Show as Table", "/big-query/show-table") htmlfrag.Wb(b1, "Show as Chart", "/big-query/show-chart") htmlfrag.Wb(b1, "As HTML", "/big-query/html") htmlfrag.Wb(b1, "Namespaces + Task Queues", "") htmlfrag.Wb(b1, "Increment", "/namespaced-counters/increment") htmlfrag.Wb(b1, "Read", "/namespaced-counters/read") htmlfrag.Wb(b1, "Push to task-queue", "/namespaced-counters/queue-push") htmlfrag.Wb(b1, "URLs with/without ancestors", "nobr") htmlfrag.Wb(b1, "Backend", "/save-url/backend") htmlfrag.Wb(b1, "Guest Book", "") htmlfrag.Wb(b1, "Eintrag hinzufügen", "/guest-entry") htmlfrag.Wb(b1, "Einträge auflisten", "/guest-view") htmlfrag.Wb(b1, "Einträge auflisten - paged - serialized cursor", "/guest-view-cursor") b1.WriteString("<hr>\n") uiDsFs := webapi.BackendUIRendered() b1.Write(uiDsFs.Bytes()) b1.WriteString("<hr>\n") b1.Write(upload.BackendUIRendered().Bytes()) b1.Write(repo.BackendUIRendered().Bytes()) b1.Write(dedup.BackendUIRendered().Bytes()) b1.Write(coinbase.BackendUIRendered().Bytes()) b1.Write(tplx.BackendUIRendered().Bytes()) b1.Write(login.BackendUIRendered().Bytes()) b1.WriteString("<br>\n") b1.WriteString("<hr>\n") urlLocalAdmin := fmt.Sprintf("http://localhost:%v/mail", routes.DevAdminPort()) ancLocalAdmin := fmt.Sprintf(" <a target='_gae' href='%v' >local app console</a><br>\n", urlLocalAdmin) b1.WriteString(ancLocalAdmin) urlConsole := fmt.Sprintf("https://console.developers.google.com/project/%v", routes.AppID()) ancConsole := fmt.Sprintf("<a target='_gae' href='%v' ><b>global</b> developer console</a>\n", urlConsole) b1.WriteString(ancConsole) urlOldAdmin := fmt.Sprintf("https://appengine.google.com/settings?&app_id=s~%v", routes.AppID()) ancOldAdmin := fmt.Sprintf(" <a target='_gae' href='%v' >old admin UI</a><br>\n ", urlOldAdmin) b1.WriteString(ancOldAdmin) b1.WriteString(` <a target='_gae' href='http://go-lint.appspot.com/github.com/pbberlin/tools/dsu' >lint a package</a><br>`) dir := m["dir"].(string) base := m["base"].(string) b1.WriteString("<br>\n") b1.WriteString("Dir: --" + dir + "-- Base: --" + base + "-- <br>\n") b1.WriteString("<br>\n") s := fmt.Sprintf("IntegerSequenes a, b: %v %v %v<br>\n", util.MyIntSeq01(), util.MyIntSeq01(), util.MyIntSeq02()) b1.WriteString(s) // b1.WriteString("<br>\n") // b1.WriteString(fmt.Sprintf("Temp dir is %s<br>\n", os.TempDir())) b1.WriteString("<br>\n") io.WriteString(b1, "Date: "+util.TimeMarker()+" - ") b1.WriteString(fmt.Sprintf("Last Month %q - 24 Months ago is %q<br>\n", util.MonthsBack(0), util.MonthsBack(24))) b1.WriteString("<br>\n") x1 := " z" + stringspb.IncrementString("--z") x2 := " Z" + stringspb.IncrementString("--Z") x3 := " 9" + stringspb.IncrementString("--9") x4 := stringspb.IncrementString(" --Peter") sEnc := "Łódź < " + stringspb.IncrementString("Łódź") + x1 + x2 + x3 + x4 b1.WriteString(fmt.Sprint(string([]byte(sEnc)), "<br>")) b1.WriteString(tplx.Foot) w.Write(b1.Bytes()) }
func backend(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { w.Header().Set("Content-type", "text/html; charset=utf-8") w.WriteHeader(http.StatusOK) b1 := new(bytes.Buffer) util.Wb(b1, "Diverse", "") util.Wb(b1, "Login", "/login") util.Wb(b1, "Schreib-Methoden", "/write-methods") util.Wb(b1, "Letzte Email", "/email-view") util.Wb(b1, "Blob List", "/blob/list") util.Wb(b1, "Template Demo 1", "/tpl/demo1") util.Wb(b1, "Template Demo 2", "/tpl/demo2") util.Wb(b1, "Http fetch", "/fetch-url") util.Wb(b1, "Instance Info", "/instance-info/view") util.Wb(b1, "Gob encode decode", "/big-query/test-gob-codec") util.Wb(b1, "JSON encode", "/json-encode") util.Wb(b1, "JSON decode", "/json-decode") util.Wb(b1, "Fulltext put", "/fulltext-search/put") util.Wb(b1, "Fulltext get", "/fulltext-search/get") util.Wb(b1, "datastore object view quoted printabe", "/dsu/show") util.Wb(b1, "Guest Book", "") util.Wb(b1, "Eintrag hinzufügen", "/guest-entry") util.Wb(b1, "Einträge auflisten", "/guest-view") util.Wb(b1, "Einträge auflisten - paged - serialized cursor", "/guest-view-cursor") util.Wb(b1, " ", "") util.Wb(b1, "Drawing a static chart", "/image/draw-lines-example") util.Wb(b1, "Big Query ...", "") util.Wb(b1, "Get real data", "/big-query/query-into-datastore") util.Wb(b1, "Get mocked data", "/big-query/mock-data-into-datastore") util.Wb(b1, " ... with Chart", "") util.Wb(b1, "Process Data 1 (mock=1)", "/big-query/regroup-data-01?mock=0") util.Wb(b1, "Process Data 2", "/big-query/regroup-data-02?f=table") util.Wb(b1, "Show as Table", "/big-query/show-table") util.Wb(b1, "Show as Chart", "/big-query/show-chart") util.Wb(b1, "As HTML", "/big-query/html") util.Wb(b1, "Request Images ", "") util.Wb(b1, "WrapBlob from Datastore", "/image/img-from-datastore?p=chart1") util.Wb(b1, "base64 from Datastore", "/image/base64-from-datastore?p=chart1") util.Wb(b1, "base64 from Variable", "/image/base64-from-var?p=1") util.Wb(b1, "base64 from File", "/image/base64-from-file?p=static/pberg1.png") util.Wb(b1, "Namespaces + Task Queues", "") util.Wb(b1, "Increment", "/namespaced-counters/increment") util.Wb(b1, "Read", "/namespaced-counters/read") util.Wb(b1, "Push to task-queue", "/namespaced-counters/queue-push") util.Wb(b1, "URLs with/without ancestors", "") util.Wb(b1, "Backend", "/save-url/backend") util.Wb(b1, "Statistics", "/_ah/stats") b1.WriteString("<br>\n") b1.WriteString("<hr>\n") b1.WriteString("<a target='_gae' href='https://console.developers.google.com/project/347979071940' ><b>global</b> developer console</a><br>\n") b1.WriteString(" <a target='_gae' href='http://localhost:8000/mail' >app console local</a><br>\n") b1.WriteString(" <a target='_gae' href='https://appengine.google.com/settings?&app_id=s~libertarian-islands' >app console online</a><br>\n") b1.WriteString(` <a target='_gae' href='http://go-lint.appspot.com/github.com/pbberlin/tools/dsu' >lint package</a><br>`) b1.WriteString("<br>\n") b1.WriteString("<a target='_gae' href='http://localhost:8085/' >app local</a><br>\n") b1.WriteString("<a target='_gae_r' href='http://libertarian-islands.appspot.com/' >app online</a><br>\n") dir := m["dir"].(string) base := m["base"].(string) b1.WriteString("<br>\n") b1.WriteString("Dir: --" + dir + "-- Base: --" + base + "-- <br>\n") b1.WriteString("<br>\n") s := fmt.Sprintf("IntegerSequenes a, b: %v %v %v<br>\n", util_err.MyIntSeq01(), util_err.MyIntSeq01(), util_err.MyIntSeq02()) b1.WriteString(s) // b1.WriteString("<br>\n") // b1.WriteString(fmt.Sprintf("Temp dir is %s<br>\n", os.TempDir())) b1.WriteString("<br>\n") b2 := new(bytes.Buffer) b2.WriteString("data:image/png;base64,...") b1.WriteString(fmt.Sprintf("Mime from %q is %q<br>\n", b2.String(), conv.MimeFromBase64(b2))) b1.WriteString("<br>\n") io.WriteString(b1, "Date: "+util.TimeMarker()+" - ") b1.WriteString(fmt.Sprintf("Last Month %q - 24 Months ago is %q<br>\n", util.MonthsBack(0), util.MonthsBack(24))) b1.WriteString("<br>\n") x1 := " z" + util.IncrementString("--z") x2 := " Z" + util.IncrementString("--Z") x3 := " 9" + util.IncrementString("--9") x4 := " Peter" + util.IncrementString("--Peter") sEnc := "Theo - wir fahrn nach Łódź < " + util.IncrementString("Łódź") + x1 + x2 + x3 + x4 b1.WriteString(fmt.Sprint("restore string string( []byte(sEnc) ): ", string([]byte(sEnc)), "<br>")) w.Header().Set("Content-Type", "text/html") w.Write(b1.Bytes()) }