// Increment increments the named counter.
func Increment(c context.Context, valName string) error {
// Get counter config.
shardsTotal := dsu.WrapInt{}
dsu.McacheGet(c, mcKeyShardsTotal(valName), &shardsTotal)
if shardsTotal.I < 1 {
ckey := datastore.NewKey(c, dsKindNumShards, mcKeyShardsTotal(valName), 0, nil)
errTx := datastore.RunInTransaction(c,
func(c context.Context) error {
err := datastore.Get(c, ckey, &shardsTotal)
if err == datastore.ErrNoSuchEntity {
shardsTotal.I = defaultNumShards
_, err = datastore.Put(c, ckey, &shardsTotal)
}
return err
}, nil)
if errTx != nil {
return errTx
}
dsu.McacheSet(c, mcKeyShardsTotal(valName), dsu.WrapInt{shardsTotal.I})
}
// pick random counter and increment it
errTx := datastore.RunInTransaction(c,
func(c context.Context) error {
shardId := rand.Intn(shardsTotal.I)
dsKey := datastore.NewKey(c, dsKindShard, keySingleShard(valName, shardId), 0, nil)
var sd WrapShardData
err := datastore.Get(c, dsKey, &sd)
// A missing entity and a present entity will both work.
if err != nil && err != datastore.ErrNoSuchEntity {
return err
}
sd.Name = valName
sd.ShardId = shardId
sd.I++
_, err = datastore.Put(c, dsKey, &sd)
if ll > 2 {
aelog.Infof(c, "ds put %v %v", dsKey, sd)
}
return err
}, nil)
if errTx != nil {
return errTx
}
memcache.Increment(c, mcKey(valName), 1, 0)
// collect number of updates
// per valName per instance in memory
// for every interval of 10 minutes
//
// a batch job checks if the number of shards should be increased or decreased
// and truncates this map
updateSamplingFrequency[valName+util.TimeMarker()[:len("2006-01-02 15:0")]] += 1
return nil
}
/* Domain is different!!! appspotMAIL.com not appspot.com peter@[email protected] https://developers.google.com/appengine/docs/python/mail/receivingmail email-address: [email protected] is routed to /_ah/mail/[email protected] [email protected] is routed to /_ah/mail/[email protected] */ func emailReceiveAndStore(w http.ResponseWriter, r *http.Request, mx map[string]interface{}) { c := appengine.NewContext(r) defer r.Body.Close() msg, err := go_mail.ReadMessage(r.Body) loghttp.E(w, r, err, false, "could not do ReadMessage") if msg == nil { aelog.Warningf(c, "-empty msg- "+r.URL.Path) return } // see http://golang.org/pkg/net/mail/#Message b1 := new(bytes.Buffer) // for i, m1 := range msg.Header { // aelog.Infof(c,"--msg header %q : %v", i, m1) // } from := msg.Header.Get("from") + "\n" b1.WriteString("from: " + from) to := msg.Header.Get("to") + "\n" b1.WriteString("to: " + to) subject := msg.Header.Get("subject") + "\n" b1.WriteString("subject: " + subject) when, _ := msg.Header.Date() swhen := when.Format("2006-01-02 - 15:04 \n") b1.WriteString("when: " + swhen) ctype := msg.Header.Get("Content-Type") aelog.Infof(c, "content type header: %q", ctype) boundary := "" // [multipart/mixed; boundary="------------060002090509030608020402"] if strings.HasPrefix(ctype, "[multipart/mixed") || strings.HasPrefix(ctype, "multipart/mixed") { vT1 := strings.Split(ctype, ";") if len(vT1) > 1 { aelog.Infof(c, "substring 1: %q", vT1[1]) sT11 := vT1[1] sT11 = strings.TrimSpace(sT11) sT11 = strings.TrimPrefix(sT11, "boundary=") sT11 = strings.Trim(sT11, `"`) boundary = sT11 aelog.Infof(c, "substring 2: %q", boundary) } } b1.WriteString("\n\n") b1.ReadFrom(msg.Body) dsu.McacheSet(c, keyLatest, dsu.WrapBlob{Name: subject, S: boundary, VByte: b1.Bytes()}) if strings.HasPrefix(to, "foscam") { // send confirmation to sender var m map[string]string = nil m = make(map[string]string) m["sender"] = from m["subject"] = "confirmation: " + subject emailSend(w, r, m) parseFurther(w, r, true) call(w, r, mx) } else { blob := dsu.WrapBlob{Name: subject + "from " + from + "to " + to, S: boundary, VByte: b1.Bytes()} blob.VVByte, _ = conv.String_to_VVByte(b1.String()) dsu.BufPut(c, blob, "email-"+util.TimeMarker()) } }
// Count retrieves the value of the named counter.
// Either from memcache - or from datastore
func Count(c context.Context, valName string) (retVal int, err error) {
wi := dsu.WrapInt{}
errMc := dsu.McacheGet(c, mcKey(valName), &wi)
if errMc == false {
aelog.Errorf(c, "%v", errMc)
}
retVal = wi.I
if retVal > 0 {
if ll > 2 {
aelog.Infof(c, "found counter %s = %v in memcache; return", mcKey(valName), wi.I)
}
retVal = 0
}
Loop1:
for j := 0; j < 1333; j++ {
q := datastore.NewQuery(dsKindShard)
q = q.Filter("Name =", valName)
// because we have "hashed" the keys, we can no longer
// range query them by key -
//q = q.Filter("__key__ >=", valName+shardId )
//q = q.Filter("__key__ < ",stringspb.IncrementString(valName+shardId) )
q = q.Order("Name")
q = q.Order("-ShardId")
q = q.Limit(-1)
q = q.Limit(batchSize)
q = q.Offset(j * batchSize)
cntr := 0
iter := q.Run(c)
for {
var sd WrapShardData
_, err = iter.Next(&sd)
if err == datastore.Done {
if ll > 2 {
aelog.Infof(c, " No Results (any more) %v", err)
}
err = nil
if cntr == 0 {
if ll > 2 {
aelog.Infof(c, " Leaving Loop1")
}
break Loop1
}
break
}
cntr++
retVal += sd.I
if ll > 2 {
aelog.Infof(c, " %2vth shard: %v %v %4v - %4v", cntr, sd.Name, sd.ShardId, sd.I, retVal)
}
}
if ll > 2 {
aelog.Infof(c, " %2v shards found - sum %4v", cntr, retVal)
}
}
dsu.McacheSet(c, mcKey(valName), retVal)
return
}
// Count retrieves the value of the named counter.
// Either from memcache - or from datastore
func Count(w http.ResponseWriter, r *http.Request, valName string) (retVal int, err error) {
c := appengine.NewContext(r)
wi := dsu.WrapInt{}
errMc := dsu.McacheGet(c, mCKValue(valName), &wi)
util_err.Err_http(w, r, errMc, false)
retVal = wi.I
if retVal > 0 {
c.Infof("found counter %s = %v in memcache; return", mCKValue(valName), wi.I)
retVal = 0
//return
}
Loop1:
for j := 0; j < 1333; j++ {
q := datastore.NewQuery(dsKindShard)
q = q.Filter("Name =", valName)
// because we have "hashed" the keys, we can no longer
// range query them by key -
//q = q.Filter("__key__ >=", valName+shardId )
//q = q.Filter("__key__ < ",util.IncrementString(valName+shardId) )
q = q.Order("Name")
q = q.Order("-ShardId")
q = q.Limit(-1)
q = q.Limit(batchSize)
//q = q.Offset(0)
q = q.Offset(j * batchSize)
cntr := 0
iter := q.Run(c)
for {
var sd WrapShardData
_, err = iter.Next(&sd)
if err == datastore.Done {
c.Infof(" No Results (any more) %v", err)
err = nil
if cntr == 0 {
c.Infof(" Leaving Loop1")
break Loop1
}
break
}
cntr++
retVal += sd.I
c.Infof(" %2vth shard: %v %v %4v - %4v", cntr, sd.Name, sd.ShardId, sd.I, retVal)
util_err.Err_http(w, r, err, false)
// other err
// if err != nil {
// return retVal, err
// }
}
c.Infof(" %2v shards found - sum %4v", cntr, retVal)
}
dsu.McacheSet(c, mCKValue(valName), retVal)
return
}
