// CacheDevice puts the specified Device into the cache (memcache).
func CacheDevice(c appengine.Context, dev *ds.Device) {
mk := prefixDevForRandID + dev.RandID
if err := memcache.Set(c, &memcache.Item{Key: mk, Value: dev.Encode()}); err != nil {
c.Warningf("Failed to set %s in memcache: %v", mk, err)
}
}
// genNewID generates a new, unique device ID
func genNewRandID(p *page.Params, d *ds.Device) {
b := make([]byte, 9) // Use 9 bytes: multiple of 3 bytes (ideal for base64 encoding so no padding '=' signs will be needed)
if _, p.Err = rand.Read(b); p.Err != nil {
return
}
RandID := base64.URLEncoding.EncodeToString(b)
// Check if RandID is unique.
// It will be, but once in a million years we might maybe perhaps encounter a match!
q := datastore.NewQuery(ds.ENameDevice).Filter(ds.PNameRandID+"=", RandID).KeysOnly().Limit(1)
var devKeys []*datastore.Key
if devKeys, p.Err = q.GetAll(p.AppCtx, nil); p.Err != nil {
return
}
if len(devKeys) > 0 {
p.Err = fmt.Errorf("Generated device RandID already exists: %s", RandID)
return
}
d.RandID = RandID
}
// logs is the logic implementation of the Logs page.
func logs(p *page.Params) {
c := p.AppCtx
// First get devices
var devices []*ds.Device
if devices, p.Err = cache.GetDevListForAccKey(c, p.Account.GetKey(c)); p.Err != nil {
return
}
p.Custom["Devices"] = devices
fv := p.Request.FormValue
p.Custom["Before"] = fv("before")
p.Custom["After"] = fv("after")
p.Custom["SearchLoc"] = fv("loc")
if fv("devID") == "" {
// No device chosen yet
return
}
var err error
var devID int64
if devID, err = strconv.ParseInt(string(fv("devID")), 10, 64); err != nil {
p.ErrorMsg = "Invalid Device! Please select a Device from the list below."
return
}
// Check if device is owned by the user:
var dev *ds.Device
for _, d := range devices {
if d.KeyID == devID {
dev = d
p.Custom["Device"] = d
break
}
}
if dev == nil {
p.ErrorMsg = "You do not have access to the specified Device! Please select a Device from the list below."
return
}
// Parse filters:
var before time.Time
if fv("before") != "" {
if before, err = p.ParseTime(timeLayout, strings.TrimSpace(fv("before"))); err != nil {
p.ErrorMsg = template.HTML(`Invalid <span class="highlight">Before</span>!`)
return
}
// Add 1 second to the parsed time because fraction of a second is not parsed but exists,
// so this new time will also include records which has the same time up to the second part and has millisecond part too.
before = before.Add(time.Second)
}
var after time.Time
if fv("after") != "" {
if after, err = p.ParseTime(timeLayout, strings.TrimSpace(fv("after"))); err != nil {
p.ErrorMsg = template.HTML(`Invalid <span class="highlight">After</span>!`)
return
}
}
var searchLoc appengine.GeoPoint
areaCode := int64(-1)
if dev.Indexed() && fv("loc") != "" {
// GPS coordinates; lat must be in range -90..90, lng must be in range -180..180
baseErr := template.HTML(`Invalid <span class="highlight">Location</span>!`)
var coords = strings.Split(strings.TrimSpace(fv("loc")), ",")
if len(coords) != 2 {
p.ErrorMsg = baseErr
return
}
searchLoc.Lat, err = strconv.ParseFloat(coords[0], 64)
if err != nil {
p.ErrorMsg = baseErr
return
}
searchLoc.Lng, err = strconv.ParseFloat(coords[1], 64)
if err != nil {
p.ErrorMsg = baseErr
return
}
if !searchLoc.Valid() {
p.ErrorMsg = template.HTML(`Invalid <span class="highlight">Location</span> specified by latitude and longitude! Valid range: [-90, 90] latitude and [-180, 180] longitude`)
return
}
areaCode = AreaCodeForGeoPt(dev.AreaSize, searchLoc.Lat, searchLoc.Lng)
}
var page int
cursorsString := fv("cursors")
var cursors = strings.Split(cursorsString, ";")[1:] // Split always returns at least 1 element (and we use semicolon separator before cursors)
// Form values
if fv("page") == "" {
page = 1
} else {
page, err = strconv.Atoi(fv("page"))
if err != nil || page < 1 {
page = 1
}
if page > len(cursors) { // If page is provided, so are (should be) the cursors
page = len(cursors)
}
}
switch {
case fv("submitFirstPage") != "":
page = 1
case fv("submitPrevPage") != "":
if page > 1 {
page--
}
case fv("submitNextPage") != "":
page++
}
pageSize := p.Account.GetLogsPageSize()
if ps := fv("pageSize"); ps != "" && ps != strconv.Itoa(pageSize) {
// Page size has been changed (on Settings page), drop cursors, render page 1
page = 1
cursorsString = ""
cursors = make([]string, 0, 1)
}
// 'ts all good, proceed with the query:
q := datastore.NewQuery(ds.ENameGPS).Filter(ds.PNameDevKeyID+"=", devID)
if !before.IsZero() {
q = q.Filter(ds.PNameCreated+"<", before)
}
if !after.IsZero() {
q = q.Filter(ds.PNameCreated+">", after)
}
if areaCode >= 0 {
q = q.Filter(ds.PNameAreaCodes+"=", areaCode)
}
q = q.Order("-" + ds.PNameCreated).Limit(pageSize)
var records = make([]*ds.GPS, 0, pageSize)
// If there is a cursor, set it.
// Page - cursor index mapping: cursors[page-2]
// 1st page: no cursor, 2nd page: cursors[0], 3nd page: cursors[1], ...
if page > 1 && page <= len(cursors)+1 {
var cursor datastore.Cursor
if cursor, p.Err = datastore.DecodeCursor(cursors[page-2]); p.Err != nil {
return
}
q = q.Start(cursor)
}
// Iterate over the results:
t := q.Run(c)
for {
r := new(ds.GPS)
_, err := t.Next(r)
if err == datastore.Done {
break
}
if err != nil {
// Datastore error
p.Err = err
return
}
records = append(records, r)
r.Dd = -1 // For now, will be set if applicable
if r.Track() {
// Check the previous (in time) record and calculate distance.
// If previous is not a Track, check the one before that etc.
for i := len(records) - 2; i >= 0; i-- {
if prev := records[i]; prev.Track() {
prev.Dd = Distance(r.GeoPoint.Lat, r.GeoPoint.Lng, prev.GeoPoint.Lat, prev.GeoPoint.Lng)
prev.Dt = prev.Created.Sub(r.Created)
break
}
}
}
}
if len(records) == 0 {
// End of list reached, disable Next page button:
p.Custom["EndOfList"] = true
}
if page == 1 || page > len(cursors) {
// Get updated cursor and store it for next page:
var cursor datastore.Cursor
if cursor, p.Err = t.Cursor(); p.Err != nil {
return
}
cursorString := cursor.String()
if page == 1 {
// If new records were inserted, they appear on the first page in which case
// the cursor for the 2nd page changes (and all other cursors will change).
// In this case drop all the cursors:
if len(cursors) > 0 && cursors[0] != cursorString {
cursorsString = ""
cursors = make([]string, 0, 1)
}
} else {
// When end of list is reached, the same cursor will be returned
if len(records) == 0 && page == len(cursors)+1 && cursors[page-2] == cursorString {
// Add 1 extra, empty page, but not more.
if page > 2 && cursors[page-3] == cursorString {
// An extra, empty page has already been added, do not add more:
page--
}
}
}
if page > len(cursors) {
cursors = append(cursors, cursorString)
cursorsString += ";" + cursorString
}
}
// Calculate labels: '1'..'9' then 'A'...
for i, lbl := len(records)-1, '1'; i >= 0; i-- {
if r := records[i]; r.Track() {
r.Label = lbl
if lbl == '9' {
lbl = 'A' - 1
}
lbl++
}
}
p.Custom["CursorList"] = cursors
p.Custom["Cursors"] = cursorsString
p.Custom["Page"] = page
p.Custom["PageSize"] = pageSize
p.Custom["RecordOffset"] = (page-1)*pageSize + 1
p.Custom["Records"] = records
if p.Mobile {
p.Custom["MapWidth"], p.Custom["MapHeight"] = p.Account.GetMobMapPrevSize()
p.Custom["MapImgFormat"] = p.Account.GetMobMapImgFormat()
} else {
p.Custom["MapWidth"], p.Custom["MapHeight"] = p.Account.GetMapPrevSize()
}
p.Custom["APIKey"] = "AIzaSyCEU_tZ1n0-mMg4woGKIfPqdbi0leSKvjg"
p.Custom["AllMarkers"] = allMarkers(records)
if len(records) == 0 {
if page == 1 {
if before.IsZero() && after.IsZero() && areaCode < 0 {
p.Custom["PrintNoRecordsForDev"] = true
} else {
p.Custom["PrintNoMatchForFilters"] = true
}
} else {
p.Custom["PrintNoMoreRecords"] = true
}
}
}
// devices is the logic implementation of the Devices page.
func devices(p *page.Params) {
c := p.AppCtx
fv := p.Request.PostFormValue
// Initial values:
p.Custom["SearchPrecision"] = 1000
p.Custom["LogsRetention"] = 60
// Detect form submits:
switch {
case fv("submitAdd") != "":
// Add New Device form submitted!
// Checks:
switch {
case !checkName(p, fv("name")):
case !checkSearchPrecision(p, fv("searchPrecision")):
case !checkLogsRetention(p, fv("logsRetention")):
}
if p.ErrorMsg == nil {
// All data OK, save new Device
searchPrecision, _ := strconv.ParseInt(fv("searchPrecision"), 10, 64)
logsRetention, _ := strconv.Atoi(fv("logsRetention"))
dev := ds.Device{fv("name"), 0, logsRetention, "", time.Now(), 0}
dev.SetSearchPrecision(searchPrecision)
genNewRandID(p, &dev)
if p.Err != nil {
return
}
if _, p.Err = datastore.Put(c, datastore.NewIncompleteKey(c, ds.ENameDevice, p.Account.GetKey(c)), &dev); p.Err != nil {
return // Datastore error
}
p.InfoMsg = "New Device saved successfully."
// Clear from memcache:
cache.ClearDevListForAccKey(c, p.Account.GetKey(c))
} else {
// Submitted values
p.Custom["Name"] = fv("name")
p.Custom["SearchPrecision"] = fv("searchPrecision")
p.Custom["LogsRetention"] = fv("logsRetention")
}
case fv("submitRename") != "":
// Rename Device form submitted!
if !checkName(p, fv("name")) {
break
}
if devID, err := strconv.ParseInt(string(fv("devID")), 10, 64); err != nil {
p.ErrorMsg = "Invalid Device!"
} else {
devKey := datastore.NewKey(c, ds.ENameDevice, "", devID, p.Account.GetKey(c))
var dev ds.Device
if err = datastore.Get(c, devKey, &dev); err != nil {
if err == datastore.ErrNoSuchEntity {
p.ErrorMsg = "You do not have access to the specified Device!"
} else {
// Real datastore error
p.Err = err
return
}
} else {
// Proceed to rename
dev.Name = fv("name")
if _, p.Err = datastore.Put(c, devKey, &dev); p.Err != nil {
return // Datastore error
}
p.InfoMsg = "Device renamed successfully."
// Clear from memcache:
cache.ClearDevListForAccKey(c, p.Account.GetKey(c))
cache.ClearDeviceForRandID(c, dev.RandID)
dev.KeyID = devID // This is important (device is loaded freshly and not yet set)!
cache.CacheDevice(c, &dev)
}
}
case fv("submitGenNewKey") != "":
// Generate New Key form submitted!
if devID, err := strconv.ParseInt(string(fv("devID")), 10, 64); err != nil {
p.ErrorMsg = "Invalid Device!"
} else {
devKey := datastore.NewKey(c, ds.ENameDevice, "", devID, p.Account.GetKey(c))
var dev ds.Device
if err = datastore.Get(c, devKey, &dev); err != nil {
if err == datastore.ErrNoSuchEntity {
p.ErrorMsg = "You do not have access to the specified Device!"
} else {
// Real datastore error
p.Err = err
return
}
} else {
// Proceed to generate new key
// Store old RandID to remove it from cache if saving succeeds
oldRandID := dev.RandID
genNewRandID(p, &dev)
if p.Err != nil {
return
}
if _, p.Err = datastore.Put(c, devKey, &dev); p.Err != nil {
return // Datastore error
}
p.InfoMsg = template.HTML("New Key generated successfully.")
p.ImportantMsg = template.HTML("<b>Important!</b> You have to update the URL in the client application else further GPS tracking calls will be discarded!")
cache.ClearDeviceForRandID(c, oldRandID)
dev.KeyID = devID // This is important (device is loaded freshly and not yet set)!
cache.CacheDevice(c, &dev)
}
}
}
q := datastore.NewQuery(ds.ENameDevice).Ancestor(p.Account.GetKey(c)).Order(ds.PNameName)
var devices []*ds.Device
var devKeys []*datastore.Key
if devKeys, p.Err = q.GetAll(c, &devices); p.Err != nil {
return
}
for i := range devices {
devices[i].KeyID = devKeys[i].IntID()
}
p.Custom["Devices"] = devices
}
// gpsHandler is the handler of the requests originating from (GPS) clients
// reporting GPS coordinates, start/stop events.
func gpsHandler(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
// General logs for all requests
c.Debugf("Location: %s;%s;%s;%s", r.Header.Get("X-AppEngine-Country"), r.Header.Get("X-AppEngine-Region"), r.Header.Get("X-AppEngine-City"), r.Header.Get("X-AppEngine-CityLatLong"))
// If device id is invalid, we want to return sliently and not let the client know about it.
// So first check non-user related params first, because else if the client would intentionally
// provide an invalid param and get no error, he/she would know that the device id is invalid.
RandID := r.FormValue("dev")
if RandID == "" {
c.Errorf("Missing Device ID (dev) parameter!")
http.Error(w, "Missing Device ID (dev) parameter!", http.StatusBadRequest)
return
}
// Do a check for RandID length: it is used to construct a memcache key which has a 250 bytes limit!
if len(RandID) > 100 {
c.Errorf("Invalid Device ID (dev) parameter!")
http.Error(w, "Invalid Device ID (dev) parameter!", http.StatusBadRequest)
return
}
gps := ds.GPS{Created: time.Now()}
var err error
tracker := r.FormValue("tracker")
if tracker != "" {
// Start-stop event
switch tracker {
case "start":
gps.AreaCodes = []int64{int64(ds.EvtStart)}
case "stop":
gps.AreaCodes = []int64{int64(ds.EvtStop)}
default:
c.Errorf("Invalid tracker parameter!")
http.Error(w, "Invalid tracker parameter!", http.StatusBadRequest)
return
}
} else {
// GPS coordinates; lat must be in range -90..90, lng must be in range -180..180
gps.GeoPoint.Lat, err = strconv.ParseFloat(r.FormValue("lat"), 64)
if err != nil {
c.Errorf("Missing or invalid latitude (lat) parameter!")
http.Error(w, "Missing or invalid latitude (lat) parameter!", http.StatusBadRequest)
return
}
gps.GeoPoint.Lng, err = strconv.ParseFloat(r.FormValue("lon"), 64)
if err != nil {
c.Errorf("Missing or invalid longitude (lon) parameter!")
http.Error(w, "Missing or invalid longitude (lon) parameter!", http.StatusBadRequest)
return
}
if !gps.GeoPoint.Valid() {
c.Errorf("Invalid geopoint specified by latitude (lat) and longitude (lon) parameters (valid range: [-90, 90] latitude and [-180, 180] longitude)!")
http.Error(w, "Invalid geopoint specified by latitude (lat) and longitude (lon) parameters (valid range: [-90, 90] latitude and [-180, 180] longitude)!", http.StatusBadRequest)
return
}
}
var dev *ds.Device
dev, err = cache.GetDevice(c, RandID)
if err != nil {
c.Errorf("Failed to get Device with cache.GetDevice(): %v", err)
http.Error(w, "", http.StatusInternalServerError)
return
}
if dev == nil {
// Invalid RandID. Do nothing and return silently.
return
}
gps.DevKeyID = dev.KeyID
if tracker == "" && dev.Indexed() {
gps.AreaCodes = logic.AreaCodesForGeoPt(dev.AreaSize, gps.GeoPoint.Lat, gps.GeoPoint.Lng)
}
var gpsKey *datastore.Key
if dev.DelOldLogs() {
// There is a (positive) Logs Retention for the device.
// We could delete old records, but that costs us a lot of Datastore write ops
// (exactly the same as saving a new record).
// Instead I query for old records (beyond the Logs Retention),
// and "resave" to those records (basically save a new record with an existing key).
// This way no deletion has to be paid for.
// Optimal solution would be to pick the oldest record, but that would require a new index (G: d, t)
// which is a "waste", so I just use the existing index (G: d, -t). This will result in the first record
// that is just over the Retention period.
// Load the existing record by key (strong consistency) and check if its timestamp is truely
// beyond the retention (because a concurrent resave might have happened).
// Query more than 1 record (limit>1) because if the latest is just concurrently resaved,
// we have more records without executing another query. And small operations are free (reading keys).
t := time.Now().Add(-24 * time.Hour * time.Duration(dev.LogsRetention))
q := datastore.NewQuery(ds.ENameGPS).
Filter(ds.PNameDevKeyID+"=", dev.KeyID).
Filter(ds.PNameCreated+"<", t).
Order("-" + ds.PNameCreated).KeysOnly().Limit(7)
keys, err := q.GetAll(c, nil)
if err != nil {
c.Errorf("Failed to list GPS records beyond Retention period: %v", err)
http.Error(w, "", http.StatusInternalServerError)
return
}
for _, key := range keys {
// Load by key (strongly consistent)
var oldGps ds.GPS
if err = datastore.Get(c, key, &oldGps); err != nil {
c.Errorf("Failed to load GPS by Key: %v", err)
http.Error(w, "", http.StatusInternalServerError)
return
}
if t.After(oldGps.Created) {
// Good: it is still older (not resaved concurrently). We will use this!
gpsKey = key
break
}
}
}
if gpsKey == nil {
// No current record to resave, create a new incomplete key for a new record:
gpsKey = datastore.NewIncompleteKey(c, ds.ENameGPS, nil)
}
_, err = datastore.Put(c, gpsKey, &gps)
if err != nil {
c.Errorf("Failed to store GPS record: %v", err)
http.Error(w, "", http.StatusInternalServerError)
return
}
}
