func (m *baseModel) fetchIds(conn redis.Conn) ([]string, error) {
ids, err := redis.Strings(conn.Do("SMEMBERS", m.idType+"s"))
m.log.Debugf("Found %d %s id(s)", m.idType, len(ids))
return ids, err
}
func (m *baseModel) fetch(id string, obj interface{}, syncing bool, conn redis.Conn) error {
m.log.Debugf("Fetching %s %s", m.idType, id)
/* if _, ok := m.entityLocks[id]; !ok {
m.entityLocks[id] = &sync.Mutex{}
}
m.entityLocks[id].Lock()
m.log.Debugf("Locked %s %s", m.idType, id)
defer m.entityLocks[id].Unlock()*/
item, err := redis.Values(conn.Do("HGETALL", m.idType+":"+id))
if err != nil {
return err
}
if len(item) == 0 {
return RecordNotFound
}
if err := redis.ScanStruct(item, obj); err != nil {
return err
}
if m.onFetch != nil {
err = m.onFetch(obj, syncing, conn)
}
return err
}
func (m *baseModel) save(id string, obj interface{}, conn redis.Conn) (bool, error) {
m.log.Debugf("Saving %s %s", m.idType, id)
defer syncFS()
existing := reflect.New(m.objType)
err := m.fetch(id, existing.Interface(), false, conn)
if err != nil && err != RecordNotFound {
return false, err
}
brandNew := err == RecordNotFound
if err == nil {
if m.isUnchanged(existing.Interface(), obj) {
m.log.Debugf("%s %s was unchanged.", m.idType, id)
return false, nil
// XXX: Should this be return RecordUnchanged?
}
}
args := redis.Args{}
args = args.Add(m.idType + ":" + id)
args = args.AddFlat(obj)
_, err = conn.Do("HMSET", args...)
if err != nil {
return false, fmt.Errorf("Failed to save object %s error:%s", id, err)
}
_, err = conn.Do("SADD", m.idType+"s", id)
if err != nil {
return false, fmt.Errorf("Failed to add object %s to list of ids error:%s", id, err)
}
if m.afterSave != nil {
err = m.afterSave(obj, conn)
if err != nil {
return true, fmt.Errorf("Error during afterSave callback: %s", err)
}
}
if m.sendEvent != nil {
if brandNew {
m.sendEvent("created", id)
} else {
m.sendEvent("updated", id)
}
}
return true, m.markUpdated(id, time.Now(), conn)
}
func (m *ModuleModel) SetConfig(moduleID string, config string, conn redis.Conn) error {
m.syncing.Wait()
//defer m.sync()
defer syncFS()
_, err := conn.Do("HSET", "module:"+moduleID, "config", config)
return err
}
func (m *ThingModel) deleteRelationshipWithDevice(deviceID string, conn redis.Conn) error {
defer syncFS()
_, err := conn.Do("HDEL", "device-thing", deviceID)
return err
}
func (m *ModuleModel) DeleteConfig(moduleID string, conn redis.Conn) error {
m.syncing.Wait()
//defer m.sync()
defer syncFS()
_, err := conn.Do("HDEL", "module:"+moduleID, "config")
return err
}
func (m *ThingModel) afterSave(thing *model.Thing, conn redis.Conn) error {
defer syncFS()
m.log.Debugf("afterSave - thing received id:%s with device:%s", thing.ID, thing.DeviceID)
existingDeviceID, err := m.GetDeviceIDForThing(thing.ID, conn)
if err != nil && err != RecordNotFound {
return fmt.Errorf("Failed to get existing device relationship error:%s", err)
}
if existingDeviceID != thing.DeviceID {
if thing.DeviceID == nil {
// Theres no device, so remove the existing relationship if it's there
deviceID, err := m.GetDeviceIDForThing(thing.ID, conn)
if err == nil {
err = m.deleteRelationshipWithDevice(*deviceID, conn)
}
if err != nil {
return fmt.Errorf("Failed to remove existing device relationship error:%s", err)
}
} else {
// See if another thing is already attached to the device
existingThingID, err := m.GetThingIDForDevice(thing.ID, conn)
if existingThingID != nil {
// Remove the existing relationship
err = m.deleteRelationshipWithDevice(*thing.DeviceID, conn)
if err != nil {
return fmt.Errorf("Failed to remove existing relationship to device %s. Currently attached to thing %s, we wanted it to be attached to %s. Error:%s", *thing.DeviceID, *existingThingID, thing.ID, err)
}
}
_, err = conn.Do("HSET", "device-thing", *thing.DeviceID, thing.ID)
if err != nil {
return fmt.Errorf("Failed to update device relationship. error: %s", err)
}
}
if err == nil {
err = m.markUpdated(thing.ID, time.Now(), conn)
if err != nil {
return fmt.Errorf("Failed to mark thing updated. error: %s", err)
}
}
}
return nil
}
func (m *baseModel) markUpdated(id string, t time.Time, conn redis.Conn) error {
defer syncFS()
ts, err := t.MarshalText()
if err != nil {
return err
}
_, err = conn.Do("HSET", m.idType+"s:updated", id, ts)
return err
}
func (m *baseModel) getLastUpdated(id string, conn redis.Conn) (*time.Time, error) {
timeString, err := redis.String(conn.Do("HGET", m.idType+"s:updated", id))
if err != nil || timeString == "" {
return nil, err
}
t := time.Time{}
err = t.UnmarshalText([]byte(timeString))
return &t, err
}
func (m *ModuleModel) GetConfig(moduleID string, conn redis.Conn) (*string, error) {
m.syncing.Wait()
exists, err := redis.Bool(conn.Do("HEXISTS", "module:"+moduleID, "config"))
if exists {
item, err := conn.Do("HGET", "module:"+moduleID, "config")
config, err := redis.String(item, err)
return &config, err
}
return nil, err
}
func (m *RoomModel) MoveThing(from *string, to *string, thing string, conn redis.Conn) error {
m.syncing.Wait()
//defer m.sync()
var err error
if to != nil {
_, err := m.Fetch(*to, conn) // Ensure the room we are putting it into actually exists
if err != nil {
return err
}
}
// Don't do a damn thing.
if from == to {
return nil
}
defer syncFS()
if to != nil && from == nil {
// we need to add it
dest := "room:" + *to + ":things"
m.log.Debugf("Adding thing %s to room %s", thing, *to)
_, err = conn.Do("SADD", dest, thing)
} else if from != nil && to == nil {
// we need to remove it
src := "room:" + *from + "s:things"
m.log.Debugf("Removing thing %s from room %s", thing, *from)
_, err = conn.Do("SREM", src, thing)
} else {
// need to move it
src := "room:" + *from + "s:things"
dest := "room:" + *to + ":things"
m.log.Debugf("Moving thing %s from room %s to room %s", thing, *from, *to)
result, err2 := conn.Do("SMOVE", src, dest, thing)
if err2 != nil {
return err2
}
if result == 0 {
// Wasn"t in src, we should add it to dest
_, err = conn.Do("SADD", dest, thing)
}
}
return err
}
func (m *ThingModel) GetThingIDForDevice(deviceID string, conn redis.Conn) (*string, error) {
item, err := conn.Do("HGET", "device-thing", deviceID)
if err != nil {
return nil, err
}
if item == nil {
return nil, RecordNotFound
}
thingID, err := redis.String(item, err)
return &thingID, err
}
func (m *ChannelModel) Create(deviceID string, channel *model.Channel, conn redis.Conn) error {
m.syncing.Wait()
//defer m.sync()
defer syncFS()
channel.DeviceID = deviceID
if _, err := m.save(deviceID+"-"+channel.ID, channel, conn); err != nil {
return err
}
if _, err := conn.Do("SADD", "device:"+deviceID+":channels", channel.ID); err != nil {
return err
}
return nil
}
func (m *ChannelModel) Delete(deviceID string, channelID string, conn redis.Conn) error {
m.syncing.Wait()
//defer m.sync()
defer syncFS()
err := m.delete(deviceID+"-"+channelID, conn)
if err != nil {
return err
}
_, err = conn.Do("SREM", "device:"+deviceID+":channels", channelID)
// TODO: announce deletion via MQTT
// publish(Ninja.topics.room.goodbye.room(roomId)
// publish(Ninja.topics.location.calibration.delete, {zone: roomId})
return err
}
func (m *baseModel) delete(id string, conn redis.Conn) error {
m.log.Debugf("Deleting %s %s", m.idType, id)
existing := reflect.New(m.objType).Interface()
existingErr := m.fetch(id, existing, false, conn)
if existingErr != nil && existingErr != RecordNotFound {
return fmt.Errorf("Failed fetching existing %s before delete. error:%s", m.idType, existingErr)
}
if existingErr == RecordNotFound {
lastUpdated, _ := m.getLastUpdated(id, conn)
if lastUpdated == nil {
// Hasn't ever existed...
return existingErr
}
// At this point we may have a RecordNotFound, but we may as well delete again anyway, just in case
m.log.Infof("%s id:%s appears to be already deleted, but we'll try again anyway.", m.idType, id)
}
defer syncFS()
conn.Send("MULTI")
conn.Send("SREM", m.idType+"s", id)
conn.Send("DEL", m.idType+":"+id)
_, err := conn.Do("EXEC")
if m.afterDelete != nil && existingErr == nil {
err = m.afterDelete(existing, conn)
if err != nil {
return fmt.Errorf("Failed on afterDelete: %s", err)
}
}
if m.sendEvent != nil {
m.sendEvent("deleted", id)
}
return m.markUpdated(id, time.Now(), conn)
}
func (m *ThingModel) GetDeviceIDForThing(thingID string, conn redis.Conn) (*string, error) {
allRels, err := redis.Strings(redis.Values(conn.Do("HGETALL", "device-thing")))
if err != nil {
return nil, err
}
if len(allRels) == 0 {
return nil, RecordNotFound
}
for i := 0; i < len(allRels); i += 2 {
if allRels[i+1] == thingID {
return &allRels[i], nil
}
}
return nil, RecordNotFound
}
func (m *ChannelModel) FetchAll(deviceID string, conn redis.Conn) (*[]*model.Channel, error) {
m.syncing.Wait()
ids, err := redis.Strings(conn.Do("SMEMBERS", "device:"+deviceID+":channels"))
m.log.Debugf("Found %d channel id(s) for device %s", len(ids), deviceID)
if err != nil {
return nil, err
}
channels := make([]*model.Channel, len(ids))
for i, id := range ids {
channels[i], err = m.Fetch(deviceID, id, conn)
if err != nil {
return nil, err
}
}
return &channels, nil
}
func (m *baseModel) getSyncManifest(conn redis.Conn) (*SyncManifest, error) {
var manifest SyncManifest = make(map[string]int64)
item, err := redis.Strings(conn.Do("HGETALL", m.idType+"s:updated"))
for i := 0; i < len(item); i += 2 {
t := time.Time{}
err := t.UnmarshalText([]byte(item[i+1]))
if err != nil {
return nil, err
}
manifest[item[i]] = t.UnixNano() / int64(time.Millisecond)
}
if err != nil {
return nil, err
}
return &manifest, nil
}
func (m *RoomModel) afterDelete(deletedRoom *model.Room, conn redis.Conn) error {
defer syncFS()
thingIds, err := redis.Strings(conn.Do("SMEMBERS", fmt.Sprintf("room:%s:things", deletedRoom.ID)))
for _, id := range thingIds {
err := m.ThingModel.SetLocation(id, nil, conn)
if err == RecordNotFound {
// We were out of sync, but don't really care...
continue
}
if err != nil {
m.log.Infof("Failed to fetch thing that was in a deleted room. ID: %s error: %s", id, err)
}
}
_, err = conn.Do("DEL", fmt.Sprintf("room:%s:things", deletedRoom.ID))
return err
}
// zpop pops a value from the ZSET key using WATCH/MULTI/EXEC commands.
func zpop(c redis.Conn, key string) (result string, err error) {
defer func() {
// Return connection to normal state on error.
if err != nil {
c.Do("DISCARD")
}
}()
// Loop until transaction is successful.
for {
if _, err := c.Do("WATCH", key); err != nil {
return "", err
}
members, err := redis.Strings(c.Do("ZRANGE", key, 0, 0))
if err != nil {
return "", err
}
if len(members) != 1 {
return "", redis.ErrNil
}
c.Send("MULTI")
c.Send("ZREM", key, members[0])
queued, err := c.Do("EXEC")
if err != nil {
return "", err
}
if queued != nil {
result = members[0]
break
}
}
return result, nil
}
func (m *baseModel) Exists(id string, conn redis.Conn) (bool, error) {
return redis.Bool(conn.Do("EXISTS", m.idType+":"+id))
}
func (m *baseModel) Sync(timeout time.Duration, conn redis.Conn) error {
m.syncing.Wait()
m.syncing.Add(1)
defer m.syncing.Done()
m.log.Infof("sync: Syncing %ss. Save data from cloud?:%s", m.idType, enableSyncFromCloud)
var diffList SyncDifferenceList
manifest, err := m.getSyncManifest(conn)
if err != nil {
return err
}
m.log.Debugf("sync: Sending %d %s local update times", len(*manifest), m.idType)
calcClient := m.SyncConn.Conn.GetServiceClient("$ninja/services/rpc/modelstore/calculate_sync_items")
err = calcClient.Call("modelstore.calculate_sync_items", []interface{}{m.idType, manifest}, &diffList, timeout)
if err != nil {
return fmt.Errorf("Failed calling calculate_sync_items for model %s error:%s", m.idType, err)
}
m.log.Infof("sync: Cloud requires %d %s(s), Node requires %d %s(s)", len(diffList.CloudRequires), m.idType, len(diffList.NodeRequires), m.idType)
if len(diffList.CloudRequires)+len(diffList.NodeRequires) == 0 {
// Nothing to do, we're in sync.
return nil
}
requestIds := make([]string, 0)
for id := range diffList.NodeRequires {
requestIds = append(requestIds, id)
}
requestedData := SyncDataSet{}
for id := range diffList.CloudRequires {
obj := reflect.New(m.objType).Interface()
err = m.fetch(id, obj, true, conn)
if err != nil && err != RecordNotFound {
return fmt.Errorf("Failed retrieving requested %s id:%s error:%s", m.idType, id, err)
}
if err == RecordNotFound {
obj = nil
}
lastUpdated, err := m.getLastUpdated(id, conn)
if err != nil {
return fmt.Errorf("Failed retrieving last updated time for requested %s id:%s error:%s", m.idType, id, err)
}
requestedData[id] = SyncObject{obj, lastUpdated.UnixNano() / int64(time.Millisecond)}
}
if deleteUnknownFromCloud {
for id := range diffList.NodeRequires {
if _, ok := (*manifest)[id]; !ok { // We've never heard of this, so remove it
m.log.Infof("Removing %s id:%s from cloud.", m.idType, id)
requestedData[id] = SyncObject{nil, time.Now().UnixNano() / int64(time.Millisecond)}
}
}
}
syncClient := m.SyncConn.Conn.GetServiceClient("$ninja/services/rpc/modelstore/do_sync_items")
var syncReply SyncReply
err = syncClient.Call("modelstore.do_sync_items", []interface{}{m.idType, requestedData, requestIds}, &syncReply, timeout)
if err != nil {
return fmt.Errorf("Failed calling do_sync_items for model %s error:%s", m.idType, err)
}
defer syncFS()
if enableSyncFromCloud {
for id, requestedObj := range syncReply.RequestedObjects {
obj := reflect.New(m.objType).Interface()
err := json.Unmarshal(requestedObj.Data, obj)
if err != nil {
m.log.Warningf("Failed to unmarshal requested %s id:%s error: %s", m.idType, id, err)
m.delete(id, conn)
} else if string(requestedObj.Data) == "null" {
m.log.Infof("Requested %s id:%s has been remotely deleted", m.idType, id)
m.delete(id, conn)
} else {
updated, err := m.save(id, obj, conn)
if err != nil {
return fmt.Errorf("Failed to save requested %s id:%s error: %s", m.idType, id, err)
}
if !updated {
m.log.Warningf("We requested an updated %s id:%s but it was the same as what we had.", m.idType, id)
}
}
err = m.markUpdated(id, time.Unix(0, requestedObj.LastModified*int64(time.Millisecond)), conn)
if err != nil {
m.log.Warningf("Failed to update last modified time of requested %s id:%s error: %s", m.idType, id, err)
}
}
} else {
m.log.Warningf("Ignoring sync data from cloud.")
}
if err != nil {
ts, err := time.Now().MarshalText()
if err != nil {
return err
}
_, err = conn.Do("SET", m.idType+"s:synced", ts)
}
return err
}