// Increment increments the named counter.
func Increment(c context.Context, valName string) error {
// Get counter config.
shardsTotal := dsu.WrapInt{}
dsu.McacheGet(c, mcKeyShardsTotal(valName), &shardsTotal)
if shardsTotal.I < 1 {
ckey := datastore.NewKey(c, dsKindNumShards, mcKeyShardsTotal(valName), 0, nil)
errTx := datastore.RunInTransaction(c,
func(c context.Context) error {
err := datastore.Get(c, ckey, &shardsTotal)
if err == datastore.ErrNoSuchEntity {
shardsTotal.I = defaultNumShards
_, err = datastore.Put(c, ckey, &shardsTotal)
}
return err
}, nil)
if errTx != nil {
return errTx
}
dsu.McacheSet(c, mcKeyShardsTotal(valName), dsu.WrapInt{shardsTotal.I})
}
// pick random counter and increment it
errTx := datastore.RunInTransaction(c,
func(c context.Context) error {
shardId := rand.Intn(shardsTotal.I)
dsKey := datastore.NewKey(c, dsKindShard, keySingleShard(valName, shardId), 0, nil)
var sd WrapShardData
err := datastore.Get(c, dsKey, &sd)
// A missing entity and a present entity will both work.
if err != nil && err != datastore.ErrNoSuchEntity {
return err
}
sd.Name = valName
sd.ShardId = shardId
sd.I++
_, err = datastore.Put(c, dsKey, &sd)
if ll > 2 {
aelog.Infof(c, "ds put %v %v", dsKey, sd)
}
return err
}, nil)
if errTx != nil {
return errTx
}
memcache.Increment(c, mcKey(valName), 1, 0)
// collect number of updates
// per valName per instance in memory
// for every interval of 10 minutes
//
// a batch job checks if the number of shards should be increased or decreased
// and truncates this map
updateSamplingFrequency[valName+util.TimeMarker()[:len("2006-01-02 15:0")]] += 1
return nil
}
func saveURLWithAncestor(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {
c := appengine.NewContext(r)
k := ds.NewKey(c, kindUrl, "", 0, ancKey(c))
s := util.TimeMarker()
ls := len(s)
lc := len("09-26 17:29:25")
lastURL_fictitious_1 := LastURL{"with_anc " + s[ls-lc:ls-3]}
_, err := ds.Put(c, k, &lastURL_fictitious_1)
loghttp.E(w, r, err, false)
}
func parseFurther(w http.ResponseWriter, r *http.Request, saveImages bool) {
c := appengine.NewContext(r)
b := new(bytes.Buffer)
defer func() {
w.Header().Set("Content-type", "text/plain; charset=utf-8")
w.Write(b.Bytes())
}()
// Get the item from the memcache
wb1 := new(dsu.WrapBlob)
ok := dsu.McacheGet(c, keyLatest, wb1)
loghttp.E(w, r, ok, true)
if ok {
b.WriteString(sp("name %v\n", wb1.Name))
b.WriteString(sp("S (boundary): %q\n", wb1.S))
// dumps the entire body
// b.WriteString(sp("B: %v\n", string(wb1.VByte)))
// instead we split it by multipart mime
vb := bytes.Split(wb1.VByte, []byte("--"+wb1.S))
for i, v := range vb {
h := "" // header
fn := "" // filename
s := string(v)
s = strings.Trim(s, "\r \n")
ctype := ""
b.WriteString(sp("\n___________mime boundary index %v___________\n", i))
if strings.HasPrefix(s, "Content-Type: image/png;") ||
strings.HasPrefix(s, "Content-Type: image/jpeg;") {
if start := strings.Index(s, sepHeaderContent); start > 0 {
h = s[:start]
vh := strings.Split(h, "\r\n")
for _, v := range vh {
v := strings.TrimSpace(v)
// b.WriteString("\t\t" + v + "\n")
if strings.HasPrefix(v, "name=") {
vv := strings.Split(v, "=")
fn = stringspb.LowerCasedUnderscored(vv[1])
}
}
s = s[start+len(sepHeaderContent):]
if posSemicol := strings.Index(h, ";"); posSemicol > 0 {
ctype = h[0:posSemicol]
}
}
}
if ctype == "" {
b.WriteString("unparseable: " + stringspb.Ellipsoider(s, 400))
} else {
b.WriteString(sp("\n\tctype=%v\n\t------------", ctype))
if fn != "" {
b.WriteString(sp("\n\tfilename=%v\n\t------------", fn))
}
if saveImages {
rE := resEntry{}
rE.when = util.TimeMarker()
rE.contentType = ctype
rE.fn = fn
rE.b64Img = &s
Images[reservoirRevolver%reservoirSize] = rE
reservoirRevolver++
aelog.Infof(c, "Put image into reservoir %v %v", fn, ctype)
}
}
}
}
}
func queryIntoDatastore(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {
limitUpper := util.MonthsBack(1)
limitLower := util.MonthsBack(25)
var q bq.QueryRequest = bq.QueryRequest{}
q.Query = `
SELECT
repository_language
, LEFT(repository_pushed_at,7) monthx
, CEIL( count(*)/1000) Tausend
FROM githubarchive:github.timeline
where 1=1
AND LEFT(repository_pushed_at,7) >= '` + limitLower + `'
AND LEFT(repository_pushed_at,7) <= '` + limitUpper + `'
AND repository_language in ('Go','go','Golang','golang','C','Java','PHP','JavaScript','C++','Python','Ruby')
AND type="PushEvent"
group by monthx, repository_language
order by repository_language , monthx
;
`
c := appengine.NewContext(r)
// The following client will be authorized by the App Engine
// app's service account for the provided scopes.
// "https://www.googleapis.com/auth/bigquery"
// "https://www.googleapis.com/auth/devstorage.full_control"
// 2015-06: instead of oauth2.NoContext we get a new type of context
var ctx context.Context = appengine.NewContext(r)
oauthHttpClient, err := google.DefaultClient(
ctx, "https://www.googleapis.com/auth/bigquery")
if err != nil {
log.Fatal(err)
}
bigqueryService, err := bq.New(oauthHttpClient)
loghttp.E(w, r, err, false)
fmt.Fprint(w, "s1<br>\n")
// Create a query statement and query request object
// query_data = {'query':'SELECT TOP(title, 10) as title, COUNT(*) as revision_count FROM [publicdata:samples.wikipedia] WHERE wp_namespace = 0;'}
// query_request = bigquery_service.jobs()
// Make a call to the BigQuery API
// query_response = query_request.query(projectId=PROJECT_NUMBER, body=query_data).execute()
js := bq.NewJobsService(bigqueryService)
jqc := js.Query("347979071940", &q)
fmt.Fprint(w, "s2 "+util.TimeMarker()+" <br>\n")
resp, err := jqc.Do()
loghttp.E(w, r, err, false)
rows := resp.Rows
var vVDest [][]byte = make([][]byte, len(rows))
aelog.Errorf(c, "%#v", rows)
for i0, v0 := range rows {
cells := v0.F
b_row := new(bytes.Buffer)
b_row.WriteString(fmt.Sprintf("r%0.2d -- ", i0))
for i1, v1 := range cells {
val1 := v1.V
b_row.WriteString(fmt.Sprintf("c%0.2d: %v ", i1, val1))
}
vVDest[i0] = []byte(b_row.Bytes())
}
key_combi, _ := dsu.BufPut(c, dsu.WrapBlob{Name: "bq_res1", VVByte: vVDest}, "bq_res1")
dsObj, _ := dsu.BufGet(c, key_combi)
printPlaintextTable(w, r, dsObj.VVByte)
fmt.Fprint(w, "s3 "+util.TimeMarker()+" <br>\n")
}
/* Domain is different!!! appspotMAIL.com not appspot.com peter@[email protected] https://developers.google.com/appengine/docs/python/mail/receivingmail email-address: [email protected] is routed to /_ah/mail/[email protected] [email protected] is routed to /_ah/mail/[email protected] */ func emailReceiveAndStore(w http.ResponseWriter, r *http.Request, mx map[string]interface{}) { c := appengine.NewContext(r) defer r.Body.Close() msg, err := go_mail.ReadMessage(r.Body) loghttp.E(w, r, err, false, "could not do ReadMessage") if msg == nil { aelog.Warningf(c, "-empty msg- "+r.URL.Path) return } // see http://golang.org/pkg/net/mail/#Message b1 := new(bytes.Buffer) // for i, m1 := range msg.Header { // aelog.Infof(c,"--msg header %q : %v", i, m1) // } from := msg.Header.Get("from") + "\n" b1.WriteString("from: " + from) to := msg.Header.Get("to") + "\n" b1.WriteString("to: " + to) subject := msg.Header.Get("subject") + "\n" b1.WriteString("subject: " + subject) when, _ := msg.Header.Date() swhen := when.Format("2006-01-02 - 15:04 \n") b1.WriteString("when: " + swhen) ctype := msg.Header.Get("Content-Type") aelog.Infof(c, "content type header: %q", ctype) boundary := "" // [multipart/mixed; boundary="------------060002090509030608020402"] if strings.HasPrefix(ctype, "[multipart/mixed") || strings.HasPrefix(ctype, "multipart/mixed") { vT1 := strings.Split(ctype, ";") if len(vT1) > 1 { aelog.Infof(c, "substring 1: %q", vT1[1]) sT11 := vT1[1] sT11 = strings.TrimSpace(sT11) sT11 = strings.TrimPrefix(sT11, "boundary=") sT11 = strings.Trim(sT11, `"`) boundary = sT11 aelog.Infof(c, "substring 2: %q", boundary) } } b1.WriteString("\n\n") b1.ReadFrom(msg.Body) dsu.McacheSet(c, keyLatest, dsu.WrapBlob{Name: subject, S: boundary, VByte: b1.Bytes()}) if strings.HasPrefix(to, "foscam") { // send confirmation to sender var m map[string]string = nil m = make(map[string]string) m["sender"] = from m["subject"] = "confirmation: " + subject emailSend(w, r, m) parseFurther(w, r, true) call(w, r, mx) } else { blob := dsu.WrapBlob{Name: subject + "from " + from + "to " + to, S: boundary, VByte: b1.Bytes()} blob.VVByte, _ = conv.String_to_VVByte(b1.String()) dsu.BufPut(c, blob, "email-"+util.TimeMarker()) } }
func backend(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {
w.Header().Set("Content-type", "text/html; charset=utf-8")
w.WriteHeader(http.StatusOK)
if ok, _, msg := login.CheckForAdminUser(r); !ok {
w.Write([]byte(msg))
return
}
b1 := new(bytes.Buffer)
b1.WriteString(tplx.ExecTplHelper(tplx.Head, map[string]interface{}{"HtmlTitle": "Backend V1"}))
htmlfrag.Wb(b1, "Debug pprof", "/debug/pprof")
htmlfrag.Wb(b1, "Diverse", "nobr")
htmlfrag.Wb(b1, "Schreib-Methoden", "/write-methods")
htmlfrag.Wb(b1, "Letzte Email", "/email-view")
htmlfrag.Wb(b1, "Blob List", "/blob2")
htmlfrag.Wb(b1, "fetch via proxy", routes.ProxifyURI)
htmlfrag.Wb(b1, "Instance Info", "/instance-info/view")
htmlfrag.Wb(b1, "Fulltext put", "/fulltext-search/put")
htmlfrag.Wb(b1, "Fulltext get", "/fulltext-search/get")
htmlfrag.Wb(b1, "datastore object view quoted printabe", "/dsu/show")
htmlfrag.Wb(b1, "Statistics", "/_ah/stats")
htmlfrag.Wb(b1, "Request Images ", "")
htmlfrag.Wb(b1, "WrapBlob from Datastore", "/image/img-from-datastore?p=chart1")
htmlfrag.Wb(b1, "base64 from Datastore", "/image/base64-from-datastore?p=chart1")
htmlfrag.Wb(b1, "base64 from Variable", "/image/base64-from-var?p=1")
htmlfrag.Wb(b1, "base64 from File", "/image/base64-from-file?p=static/pberg1.png")
htmlfrag.Wb(b1, "Drawing a static chart", "/image/draw-lines-example")
htmlfrag.Wb(b1, "Big Query ...", "")
htmlfrag.Wb(b1, "Get real data", "/big-query/query-into-datastore")
htmlfrag.Wb(b1, "Get mocked data", "/big-query/mock-data-into-datastore")
htmlfrag.Wb(b1, " ... with Chart", "")
htmlfrag.Wb(b1, "Process Data 1 (mock=1)", "/big-query/regroup-data-01?mock=0")
htmlfrag.Wb(b1, "Process Data 2", "/big-query/regroup-data-02?f=table")
htmlfrag.Wb(b1, "Show as Table", "/big-query/show-table")
htmlfrag.Wb(b1, "Show as Chart", "/big-query/show-chart")
htmlfrag.Wb(b1, "As HTML", "/big-query/html")
htmlfrag.Wb(b1, "Namespaces + Task Queues", "")
htmlfrag.Wb(b1, "Increment", "/namespaced-counters/increment")
htmlfrag.Wb(b1, "Read", "/namespaced-counters/read")
htmlfrag.Wb(b1, "Push to task-queue", "/namespaced-counters/queue-push")
htmlfrag.Wb(b1, "URLs with/without ancestors", "nobr")
htmlfrag.Wb(b1, "Backend", "/save-url/backend")
htmlfrag.Wb(b1, "Guest Book", "")
htmlfrag.Wb(b1, "Eintrag hinzufügen", "/guest-entry")
htmlfrag.Wb(b1, "Einträge auflisten", "/guest-view")
htmlfrag.Wb(b1, "Einträge auflisten - paged - serialized cursor", "/guest-view-cursor")
b1.WriteString("<hr>\n")
uiDsFs := webapi.BackendUIRendered()
b1.Write(uiDsFs.Bytes())
b1.WriteString("<hr>\n")
b1.Write(upload.BackendUIRendered().Bytes())
b1.Write(repo.BackendUIRendered().Bytes())
b1.Write(dedup.BackendUIRendered().Bytes())
b1.Write(coinbase.BackendUIRendered().Bytes())
b1.Write(tplx.BackendUIRendered().Bytes())
b1.Write(login.BackendUIRendered().Bytes())
b1.WriteString("<br>\n")
b1.WriteString("<hr>\n")
urlLocalAdmin := fmt.Sprintf("http://localhost:%v/mail", routes.DevAdminPort())
ancLocalAdmin := fmt.Sprintf(" <a target='_gae' href='%v' >local app console</a><br>\n", urlLocalAdmin)
b1.WriteString(ancLocalAdmin)
urlConsole := fmt.Sprintf("https://console.developers.google.com/project/%v", routes.AppID())
ancConsole := fmt.Sprintf("<a target='_gae' href='%v' ><b>global</b> developer console</a>\n", urlConsole)
b1.WriteString(ancConsole)
urlOldAdmin := fmt.Sprintf("https://appengine.google.com/settings?&app_id=s~%v", routes.AppID())
ancOldAdmin := fmt.Sprintf(" <a target='_gae' href='%v' >old admin UI</a><br>\n ", urlOldAdmin)
b1.WriteString(ancOldAdmin)
b1.WriteString(` <a target='_gae'
href='http://go-lint.appspot.com/github.com/pbberlin/tools/dsu'
>lint a package</a><br>`)
dir := m["dir"].(string)
base := m["base"].(string)
b1.WriteString("<br>\n")
b1.WriteString("Dir: --" + dir + "-- Base: --" + base + "-- <br>\n")
b1.WriteString("<br>\n")
s := fmt.Sprintf("IntegerSequenes a, b: %v %v %v<br>\n", util.MyIntSeq01(), util.MyIntSeq01(), util.MyIntSeq02())
b1.WriteString(s)
// b1.WriteString("<br>\n")
// b1.WriteString(fmt.Sprintf("Temp dir is %s<br>\n", os.TempDir()))
b1.WriteString("<br>\n")
io.WriteString(b1, "Date: "+util.TimeMarker()+" - ")
b1.WriteString(fmt.Sprintf("Last Month %q - 24 Months ago is %q<br>\n", util.MonthsBack(0),
util.MonthsBack(24)))
b1.WriteString("<br>\n")
x1 := " z" + stringspb.IncrementString("--z")
x2 := " Z" + stringspb.IncrementString("--Z")
x3 := " 9" + stringspb.IncrementString("--9")
x4 := stringspb.IncrementString(" --Peter")
sEnc := "Łódź < " + stringspb.IncrementString("Łódź") + x1 + x2 + x3 + x4
b1.WriteString(fmt.Sprint(string([]byte(sEnc)), "<br>"))
b1.WriteString(tplx.Foot)
w.Write(b1.Bytes())
}
func backend(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {
w.Header().Set("Content-type", "text/html; charset=utf-8")
w.WriteHeader(http.StatusOK)
b1 := new(bytes.Buffer)
util.Wb(b1, "Diverse", "")
util.Wb(b1, "Login", "/login")
util.Wb(b1, "Schreib-Methoden", "/write-methods")
util.Wb(b1, "Letzte Email", "/email-view")
util.Wb(b1, "Blob List", "/blob/list")
util.Wb(b1, "Template Demo 1", "/tpl/demo1")
util.Wb(b1, "Template Demo 2", "/tpl/demo2")
util.Wb(b1, "Http fetch", "/fetch-url")
util.Wb(b1, "Instance Info", "/instance-info/view")
util.Wb(b1, "Gob encode decode", "/big-query/test-gob-codec")
util.Wb(b1, "JSON encode", "/json-encode")
util.Wb(b1, "JSON decode", "/json-decode")
util.Wb(b1, "Fulltext put", "/fulltext-search/put")
util.Wb(b1, "Fulltext get", "/fulltext-search/get")
util.Wb(b1, "datastore object view quoted printabe", "/dsu/show")
util.Wb(b1, "Guest Book", "")
util.Wb(b1, "Eintrag hinzufügen", "/guest-entry")
util.Wb(b1, "Einträge auflisten", "/guest-view")
util.Wb(b1, "Einträge auflisten - paged - serialized cursor", "/guest-view-cursor")
util.Wb(b1, " ", "")
util.Wb(b1, "Drawing a static chart", "/image/draw-lines-example")
util.Wb(b1, "Big Query ...", "")
util.Wb(b1, "Get real data", "/big-query/query-into-datastore")
util.Wb(b1, "Get mocked data", "/big-query/mock-data-into-datastore")
util.Wb(b1, " ... with Chart", "")
util.Wb(b1, "Process Data 1 (mock=1)", "/big-query/regroup-data-01?mock=0")
util.Wb(b1, "Process Data 2", "/big-query/regroup-data-02?f=table")
util.Wb(b1, "Show as Table", "/big-query/show-table")
util.Wb(b1, "Show as Chart", "/big-query/show-chart")
util.Wb(b1, "As HTML", "/big-query/html")
util.Wb(b1, "Request Images ", "")
util.Wb(b1, "WrapBlob from Datastore", "/image/img-from-datastore?p=chart1")
util.Wb(b1, "base64 from Datastore", "/image/base64-from-datastore?p=chart1")
util.Wb(b1, "base64 from Variable", "/image/base64-from-var?p=1")
util.Wb(b1, "base64 from File", "/image/base64-from-file?p=static/pberg1.png")
util.Wb(b1, "Namespaces + Task Queues", "")
util.Wb(b1, "Increment", "/namespaced-counters/increment")
util.Wb(b1, "Read", "/namespaced-counters/read")
util.Wb(b1, "Push to task-queue", "/namespaced-counters/queue-push")
util.Wb(b1, "URLs with/without ancestors", "")
util.Wb(b1, "Backend", "/save-url/backend")
util.Wb(b1, "Statistics", "/_ah/stats")
b1.WriteString("<br>\n")
b1.WriteString("<hr>\n")
b1.WriteString("<a target='_gae' href='https://console.developers.google.com/project/347979071940' ><b>global</b> developer console</a><br>\n")
b1.WriteString(" <a target='_gae' href='http://localhost:8000/mail' >app console local</a><br>\n")
b1.WriteString(" <a target='_gae' href='https://appengine.google.com/settings?&app_id=s~libertarian-islands' >app console online</a><br>\n")
b1.WriteString(` <a target='_gae'
href='http://go-lint.appspot.com/github.com/pbberlin/tools/dsu'
>lint package</a><br>`)
b1.WriteString("<br>\n")
b1.WriteString("<a target='_gae' href='http://localhost:8085/' >app local</a><br>\n")
b1.WriteString("<a target='_gae_r' href='http://libertarian-islands.appspot.com/' >app online</a><br>\n")
dir := m["dir"].(string)
base := m["base"].(string)
b1.WriteString("<br>\n")
b1.WriteString("Dir: --" + dir + "-- Base: --" + base + "-- <br>\n")
b1.WriteString("<br>\n")
s := fmt.Sprintf("IntegerSequenes a, b: %v %v %v<br>\n", util_err.MyIntSeq01(), util_err.MyIntSeq01(), util_err.MyIntSeq02())
b1.WriteString(s)
// b1.WriteString("<br>\n")
// b1.WriteString(fmt.Sprintf("Temp dir is %s<br>\n", os.TempDir()))
b1.WriteString("<br>\n")
b2 := new(bytes.Buffer)
b2.WriteString("data:image/png;base64,...")
b1.WriteString(fmt.Sprintf("Mime from %q is %q<br>\n", b2.String(),
conv.MimeFromBase64(b2)))
b1.WriteString("<br>\n")
io.WriteString(b1, "Date: "+util.TimeMarker()+" - ")
b1.WriteString(fmt.Sprintf("Last Month %q - 24 Months ago is %q<br>\n", util.MonthsBack(0),
util.MonthsBack(24)))
b1.WriteString("<br>\n")
x1 := " z" + util.IncrementString("--z")
x2 := " Z" + util.IncrementString("--Z")
x3 := " 9" + util.IncrementString("--9")
x4 := " Peter" + util.IncrementString("--Peter")
sEnc := "Theo - wir fahrn nach Łódź < " + util.IncrementString("Łódź") + x1 + x2 + x3 + x4
b1.WriteString(fmt.Sprint("restore string string( []byte(sEnc) ): ", string([]byte(sEnc)), "<br>"))
w.Header().Set("Content-Type", "text/html")
w.Write(b1.Bytes())
}