Ejemplo n.º 1
0
// Increment increments the named counter.
func Increment(c context.Context, valName string) error {

	// Get counter config.
	shardsTotal := dsu.WrapInt{}
	dsu.McacheGet(c, mcKeyShardsTotal(valName), &shardsTotal)
	if shardsTotal.I < 1 {
		ckey := datastore.NewKey(c, dsKindNumShards, mcKeyShardsTotal(valName), 0, nil)
		errTx := datastore.RunInTransaction(c,
			func(c context.Context) error {
				err := datastore.Get(c, ckey, &shardsTotal)
				if err == datastore.ErrNoSuchEntity {
					shardsTotal.I = defaultNumShards
					_, err = datastore.Put(c, ckey, &shardsTotal)
				}
				return err
			}, nil)
		if errTx != nil {
			return errTx
		}
		dsu.McacheSet(c, mcKeyShardsTotal(valName), dsu.WrapInt{shardsTotal.I})
	}

	// pick random counter and increment it
	errTx := datastore.RunInTransaction(c,
		func(c context.Context) error {
			shardId := rand.Intn(shardsTotal.I)
			dsKey := datastore.NewKey(c, dsKindShard, keySingleShard(valName, shardId), 0, nil)
			var sd WrapShardData
			err := datastore.Get(c, dsKey, &sd)
			// A missing entity and a present entity will both work.
			if err != nil && err != datastore.ErrNoSuchEntity {
				return err
			}
			sd.Name = valName
			sd.ShardId = shardId
			sd.I++
			_, err = datastore.Put(c, dsKey, &sd)
			if ll > 2 {
				aelog.Infof(c, "ds put %v %v", dsKey, sd)
			}
			return err
		}, nil)
	if errTx != nil {
		return errTx
	}

	memcache.Increment(c, mcKey(valName), 1, 0)

	// collect number of updates
	//    per valName per instance in memory
	//    for every interval of 10 minutes
	//
	//  a batch job checks if the number of shards should be increased or decreased
	//    and truncates this map
	updateSamplingFrequency[valName+util.TimeMarker()[:len("2006-01-02 15:0")]] += 1

	return nil
}
Ejemplo n.º 2
0
/*

	Domain is different!!!

	appspotMAIL.com

	not

	appspot.com

	peter@[email protected]


  https://developers.google.com/appengine/docs/python/mail/receivingmail

 	email-address:	 [email protected]
 	is routed to
   /_ah/mail/[email protected]

   [email protected]
 	is routed to
   /_ah/mail/[email protected]
*/
func emailReceiveAndStore(w http.ResponseWriter, r *http.Request, mx map[string]interface{}) {

	c := appengine.NewContext(r)
	defer r.Body.Close()

	msg, err := go_mail.ReadMessage(r.Body)
	loghttp.E(w, r, err, false, "could not do ReadMessage")
	if msg == nil {
		aelog.Warningf(c, "-empty msg- "+r.URL.Path)
		return
	}

	// see http://golang.org/pkg/net/mail/#Message
	b1 := new(bytes.Buffer)
	// for i, m1 := range msg.Header {
	// 	aelog.Infof(c,"--msg header %q : %v", i, m1)
	// }

	from := msg.Header.Get("from") + "\n"
	b1.WriteString("from: " + from)

	to := msg.Header.Get("to") + "\n"
	b1.WriteString("to: " + to)

	subject := msg.Header.Get("subject") + "\n"
	b1.WriteString("subject: " + subject)

	when, _ := msg.Header.Date()
	swhen := when.Format("2006-01-02 - 15:04 \n")
	b1.WriteString("when: " + swhen)

	ctype := msg.Header.Get("Content-Type")
	aelog.Infof(c, "content type header: %q", ctype)
	boundary := ""
	// [multipart/mixed; boundary="------------060002090509030608020402"]
	if strings.HasPrefix(ctype, "[multipart/mixed") ||
		strings.HasPrefix(ctype, "multipart/mixed") {
		vT1 := strings.Split(ctype, ";")
		if len(vT1) > 1 {
			aelog.Infof(c, "substring 1: %q", vT1[1])
			sT11 := vT1[1]
			sT11 = strings.TrimSpace(sT11)
			sT11 = strings.TrimPrefix(sT11, "boundary=")
			sT11 = strings.Trim(sT11, `"`)
			boundary = sT11
			aelog.Infof(c, "substring 2: %q", boundary)
		}
	}

	b1.WriteString("\n\n")
	b1.ReadFrom(msg.Body)

	dsu.McacheSet(c, keyLatest, dsu.WrapBlob{Name: subject, S: boundary, VByte: b1.Bytes()})
	if strings.HasPrefix(to, "foscam") {
		// send confirmation to sender
		var m map[string]string = nil
		m = make(map[string]string)
		m["sender"] = from
		m["subject"] = "confirmation: " + subject
		emailSend(w, r, m)

		parseFurther(w, r, true)
		call(w, r, mx)
	} else {
		blob := dsu.WrapBlob{Name: subject + "from " + from + "to " + to,
			S: boundary, VByte: b1.Bytes()}
		blob.VVByte, _ = conv.String_to_VVByte(b1.String())
		dsu.BufPut(c, blob, "email-"+util.TimeMarker())
	}

}
Ejemplo n.º 3
0
// Count retrieves the value of the named counter.
// Either from memcache - or from datastore
func Count(c context.Context, valName string) (retVal int, err error) {

	wi := dsu.WrapInt{}
	errMc := dsu.McacheGet(c, mcKey(valName), &wi)
	if errMc == false {
		aelog.Errorf(c, "%v", errMc)
	}
	retVal = wi.I
	if retVal > 0 {
		if ll > 2 {
			aelog.Infof(c, "found counter %s = %v in memcache; return", mcKey(valName), wi.I)
		}
		retVal = 0
	}

Loop1:
	for j := 0; j < 1333; j++ {

		q := datastore.NewQuery(dsKindShard)

		q = q.Filter("Name =", valName)

		// because we have "hashed" the keys, we can no longer
		// range query them by key -
		//q = q.Filter("__key__ >=", valName+shardId )
		//q = q.Filter("__key__ < ",stringspb.IncrementString(valName+shardId) )

		q = q.Order("Name")
		q = q.Order("-ShardId")
		q = q.Limit(-1)
		q = q.Limit(batchSize)
		q = q.Offset(j * batchSize)
		cntr := 0
		iter := q.Run(c)
		for {
			var sd WrapShardData
			_, err = iter.Next(&sd)

			if err == datastore.Done {
				if ll > 2 {
					aelog.Infof(c, "       No Results (any more)  %v", err)
				}
				err = nil
				if cntr == 0 {
					if ll > 2 {
						aelog.Infof(c, "  Leaving Loop1")
					}
					break Loop1
				}
				break
			}
			cntr++
			retVal += sd.I
			if ll > 2 {
				aelog.Infof(c, "        %2vth shard: %v %v %4v - %4v", cntr, sd.Name, sd.ShardId, sd.I, retVal)
			}
		}
		if ll > 2 {
			aelog.Infof(c, "   %2v shards found - sum %4v", cntr, retVal)
		}

	}

	dsu.McacheSet(c, mcKey(valName), retVal)
	return

}
Ejemplo n.º 4
0
// Count retrieves the value of the named counter.
// Either from memcache - or from datastore
func Count(w http.ResponseWriter, r *http.Request, valName string) (retVal int, err error) {

	c := appengine.NewContext(r)

	wi := dsu.WrapInt{}
	errMc := dsu.McacheGet(c, mCKValue(valName), &wi)
	util_err.Err_http(w, r, errMc, false)
	retVal = wi.I
	if retVal > 0 {
		c.Infof("found counter %s = %v in memcache; return", mCKValue(valName), wi.I)
		retVal = 0
		//return
	}

Loop1:
	for j := 0; j < 1333; j++ {

		q := datastore.NewQuery(dsKindShard)

		q = q.Filter("Name =", valName)

		// because we have "hashed" the keys, we can no longer
		// range query them by key -
		//q = q.Filter("__key__ >=", valName+shardId )
		//q = q.Filter("__key__ < ",util.IncrementString(valName+shardId) )

		q = q.Order("Name")
		q = q.Order("-ShardId")

		q = q.Limit(-1)
		q = q.Limit(batchSize)

		//q = q.Offset(0)
		q = q.Offset(j * batchSize)

		cntr := 0
		iter := q.Run(c)
		for {
			var sd WrapShardData
			_, err = iter.Next(&sd)

			if err == datastore.Done {
				c.Infof("       No Results (any more)  %v", err)
				err = nil
				if cntr == 0 {
					c.Infof("  Leaving Loop1")
					break Loop1
				}
				break
			}
			cntr++
			retVal += sd.I
			c.Infof("        %2vth shard: %v %v %4v - %4v", cntr, sd.Name, sd.ShardId, sd.I, retVal)

			util_err.Err_http(w, r, err, false)
			// other err
			// if err != nil {
			// 	return retVal, err
			// }

		}

		c.Infof("   %2v shards found - sum %4v", cntr, retVal)

	}

	dsu.McacheSet(c, mCKValue(valName), retVal)
	return

}