func (m *baseModel) fetch(id string, obj interface{}, syncing bool, conn redis.Conn) error {

	m.log.Debugf("Fetching %s %s", m.idType, id)

	/*	if _, ok := m.entityLocks[id]; !ok {
			m.entityLocks[id] = &sync.Mutex{}
		}

		m.entityLocks[id].Lock()
		m.log.Debugf("Locked %s %s", m.idType, id)
		defer m.entityLocks[id].Unlock()*/

	item, err := redis.Values(conn.Do("HGETALL", m.idType+":"+id))

	if err != nil {
		return err
	}

	if len(item) == 0 {
		return RecordNotFound
	}

	if err := redis.ScanStruct(item, obj); err != nil {
		return err
	}

	if m.onFetch != nil {
		err = m.onFetch(obj, syncing, conn)
	}

	return err
}
func (m *baseModel) fetchIds(conn redis.Conn) ([]string, error) {

	ids, err := redis.Strings(conn.Do("SMEMBERS", m.idType+"s"))
	m.log.Debugf("Found %d %s id(s)", m.idType, len(ids))

	return ids, err
}
func (m *baseModel) save(id string, obj interface{}, conn redis.Conn) (bool, error) {

	m.log.Debugf("Saving %s %s", m.idType, id)

	defer syncFS()

	existing := reflect.New(m.objType)

	err := m.fetch(id, existing.Interface(), false, conn)

	if err != nil && err != RecordNotFound {
		return false, err
	}

	brandNew := err == RecordNotFound

	if err == nil {
		if m.isUnchanged(existing.Interface(), obj) {
			m.log.Debugf("%s %s was unchanged.", m.idType, id)

			return false, nil
			// XXX: Should this be return RecordUnchanged?
		}
	}

	args := redis.Args{}
	args = args.Add(m.idType + ":" + id)
	args = args.AddFlat(obj)

	_, err = conn.Do("HMSET", args...)

	if err != nil {
		return false, fmt.Errorf("Failed to save object %s error:%s", id, err)
	}

	_, err = conn.Do("SADD", m.idType+"s", id)

	if err != nil {
		return false, fmt.Errorf("Failed to add object %s to list of ids error:%s", id, err)
	}

	if m.afterSave != nil {
		err = m.afterSave(obj, conn)

		if err != nil {
			return true, fmt.Errorf("Error during afterSave callback: %s", err)
		}
	}

	if m.sendEvent != nil {
		if brandNew {
			m.sendEvent("created", id)
		} else {
			m.sendEvent("updated", id)
		}
	}

	return true, m.markUpdated(id, time.Now(), conn)
}
func (m *ThingModel) deleteRelationshipWithDevice(deviceID string, conn redis.Conn) error {

	defer syncFS()

	_, err := conn.Do("HDEL", "device-thing", deviceID)

	return err
}
func (m *ModuleModel) DeleteConfig(moduleID string, conn redis.Conn) error {
	m.syncing.Wait()
	//defer m.sync()
	defer syncFS()

	_, err := conn.Do("HDEL", "module:"+moduleID, "config")
	return err
}
func (m *ModuleModel) SetConfig(moduleID string, config string, conn redis.Conn) error {
	m.syncing.Wait()
	//defer m.sync()
	defer syncFS()

	_, err := conn.Do("HSET", "module:"+moduleID, "config", config)
	return err
}
func (m *ThingModel) afterSave(thing *model.Thing, conn redis.Conn) error {

	defer syncFS()

	m.log.Debugf("afterSave - thing received id:%s with device:%s", thing.ID, thing.DeviceID)

	existingDeviceID, err := m.GetDeviceIDForThing(thing.ID, conn)

	if err != nil && err != RecordNotFound {
		return fmt.Errorf("Failed to get existing device relationship error:%s", err)
	}

	if existingDeviceID != thing.DeviceID {
		if thing.DeviceID == nil {

			// Theres no device, so remove the existing relationship if it's there
			deviceID, err := m.GetDeviceIDForThing(thing.ID, conn)

			if err == nil {
				err = m.deleteRelationshipWithDevice(*deviceID, conn)
			}

			if err != nil {
				return fmt.Errorf("Failed to remove existing device relationship error:%s", err)
			}

		} else {
			// See if another thing is already attached to the device
			existingThingID, err := m.GetThingIDForDevice(thing.ID, conn)

			if existingThingID != nil {
				// Remove the existing relationship
				err = m.deleteRelationshipWithDevice(*thing.DeviceID, conn)

				if err != nil {
					return fmt.Errorf("Failed to remove existing relationship to device %s. Currently attached to thing %s, we wanted it to be attached to %s. Error:%s", *thing.DeviceID, *existingThingID, thing.ID, err)
				}
			}

			_, err = conn.Do("HSET", "device-thing", *thing.DeviceID, thing.ID)

			if err != nil {
				return fmt.Errorf("Failed to update device relationship. error: %s", err)
			}
		}

		if err == nil {
			err = m.markUpdated(thing.ID, time.Now(), conn)
			if err != nil {
				return fmt.Errorf("Failed to mark thing updated. error: %s", err)
			}
		}

	}

	return nil
}
func (m *baseModel) markUpdated(id string, t time.Time, conn redis.Conn) error {
	defer syncFS()

	ts, err := t.MarshalText()
	if err != nil {
		return err
	}
	_, err = conn.Do("HSET", m.idType+"s:updated", id, ts)
	return err
}
func (m *baseModel) getLastUpdated(id string, conn redis.Conn) (*time.Time, error) {
	timeString, err := redis.String(conn.Do("HGET", m.idType+"s:updated", id))

	if err != nil || timeString == "" {
		return nil, err
	}

	t := time.Time{}
	err = t.UnmarshalText([]byte(timeString))
	return &t, err
}
func (m *ModuleModel) GetConfig(moduleID string, conn redis.Conn) (*string, error) {
	m.syncing.Wait()

	exists, err := redis.Bool(conn.Do("HEXISTS", "module:"+moduleID, "config"))

	if exists {
		item, err := conn.Do("HGET", "module:"+moduleID, "config")
		config, err := redis.String(item, err)
		return &config, err
	}

	return nil, err
}
func (m *RoomModel) MoveThing(from *string, to *string, thing string, conn redis.Conn) error {
	m.syncing.Wait()
	//defer m.sync()

	var err error

	if to != nil {
		_, err := m.Fetch(*to, conn) // Ensure the room we are putting it into actually exists

		if err != nil {
			return err
		}
	}

	// Don't do a damn thing.
	if from == to {
		return nil
	}

	defer syncFS()

	if to != nil && from == nil {
		// we need to add it

		dest := "room:" + *to + ":things"

		m.log.Debugf("Adding thing %s to room %s", thing, *to)
		_, err = conn.Do("SADD", dest, thing)

	} else if from != nil && to == nil {
		// we need to remove it

		src := "room:" + *from + "s:things"

		m.log.Debugf("Removing thing %s from room %s", thing, *from)
		_, err = conn.Do("SREM", src, thing)

	} else {
		// need to move it

		src := "room:" + *from + "s:things"
		dest := "room:" + *to + ":things"

		m.log.Debugf("Moving thing %s from room %s to room %s", thing, *from, *to)

		result, err2 := conn.Do("SMOVE", src, dest, thing)
		if err2 != nil {
			return err2
		}

		if result == 0 {
			// Wasn"t in src, we should add it to dest
			_, err = conn.Do("SADD", dest, thing)
		}

	}

	return err

}
func (m *ThingModel) GetThingIDForDevice(deviceID string, conn redis.Conn) (*string, error) {

	item, err := conn.Do("HGET", "device-thing", deviceID)

	if err != nil {
		return nil, err
	}

	if item == nil {
		return nil, RecordNotFound
	}

	thingID, err := redis.String(item, err)

	return &thingID, err
}
func (m *ChannelModel) Create(deviceID string, channel *model.Channel, conn redis.Conn) error {
	m.syncing.Wait()
	//defer m.sync()
	defer syncFS()

	channel.DeviceID = deviceID

	if _, err := m.save(deviceID+"-"+channel.ID, channel, conn); err != nil {
		return err
	}

	if _, err := conn.Do("SADD", "device:"+deviceID+":channels", channel.ID); err != nil {
		return err
	}

	return nil
}
func (m *ChannelModel) Delete(deviceID string, channelID string, conn redis.Conn) error {
	m.syncing.Wait()
	//defer m.sync()
	defer syncFS()

	err := m.delete(deviceID+"-"+channelID, conn)
	if err != nil {
		return err
	}

	_, err = conn.Do("SREM", "device:"+deviceID+":channels", channelID)

	// TODO: announce deletion via MQTT
	// publish(Ninja.topics.room.goodbye.room(roomId)
	// publish(Ninja.topics.location.calibration.delete, {zone: roomId})

	return err
}
func (m *ThingModel) GetDeviceIDForThing(thingID string, conn redis.Conn) (*string, error) {

	allRels, err := redis.Strings(redis.Values(conn.Do("HGETALL", "device-thing")))

	if err != nil {
		return nil, err
	}

	if len(allRels) == 0 {
		return nil, RecordNotFound
	}

	for i := 0; i < len(allRels); i += 2 {
		if allRels[i+1] == thingID {
			return &allRels[i], nil
		}
	}

	return nil, RecordNotFound
}
func (m *baseModel) getSyncManifest(conn redis.Conn) (*SyncManifest, error) {

	var manifest SyncManifest = make(map[string]int64)

	item, err := redis.Strings(conn.Do("HGETALL", m.idType+"s:updated"))

	for i := 0; i < len(item); i += 2 {
		t := time.Time{}
		err := t.UnmarshalText([]byte(item[i+1]))
		if err != nil {
			return nil, err
		}
		manifest[item[i]] = t.UnixNano() / int64(time.Millisecond)
	}

	if err != nil {
		return nil, err
	}

	return &manifest, nil
}
func (m *ChannelModel) FetchAll(deviceID string, conn redis.Conn) (*[]*model.Channel, error) {
	m.syncing.Wait()

	ids, err := redis.Strings(conn.Do("SMEMBERS", "device:"+deviceID+":channels"))
	m.log.Debugf("Found %d channel id(s) for device %s", len(ids), deviceID)

	if err != nil {
		return nil, err
	}

	channels := make([]*model.Channel, len(ids))

	for i, id := range ids {
		channels[i], err = m.Fetch(deviceID, id, conn)
		if err != nil {
			return nil, err
		}
	}

	return &channels, nil
}
func (m *RoomModel) afterDelete(deletedRoom *model.Room, conn redis.Conn) error {

	defer syncFS()

	thingIds, err := redis.Strings(conn.Do("SMEMBERS", fmt.Sprintf("room:%s:things", deletedRoom.ID)))

	for _, id := range thingIds {

		err := m.ThingModel.SetLocation(id, nil, conn)

		if err == RecordNotFound {
			// We were out of sync, but don't really care...
			continue
		}
		if err != nil {
			m.log.Infof("Failed to fetch thing that was in a deleted room. ID: %s error: %s", id, err)
		}

	}

	_, err = conn.Do("DEL", fmt.Sprintf("room:%s:things", deletedRoom.ID))

	return err
}
func (m *baseModel) delete(id string, conn redis.Conn) error {

	m.log.Debugf("Deleting %s %s", m.idType, id)

	existing := reflect.New(m.objType).Interface()

	existingErr := m.fetch(id, existing, false, conn)
	if existingErr != nil && existingErr != RecordNotFound {
		return fmt.Errorf("Failed fetching existing %s before delete. error:%s", m.idType, existingErr)
	}

	if existingErr == RecordNotFound {

		lastUpdated, _ := m.getLastUpdated(id, conn)
		if lastUpdated == nil {
			// Hasn't ever existed...
			return existingErr
		}
		// At this point we may have a RecordNotFound, but we may as well delete again anyway, just in case
		m.log.Infof("%s id:%s appears to be already deleted, but we'll try again anyway.", m.idType, id)
	}

	defer syncFS()

	conn.Send("MULTI")
	conn.Send("SREM", m.idType+"s", id)
	conn.Send("DEL", m.idType+":"+id)
	_, err := conn.Do("EXEC")

	if m.afterDelete != nil && existingErr == nil {
		err = m.afterDelete(existing, conn)

		if err != nil {
			return fmt.Errorf("Failed on afterDelete: %s", err)
		}
	}

	if m.sendEvent != nil {
		m.sendEvent("deleted", id)
	}

	return m.markUpdated(id, time.Now(), conn)
}
func (m *baseModel) Exists(id string, conn redis.Conn) (bool, error) {
	return redis.Bool(conn.Do("EXISTS", m.idType+":"+id))
}
// zpop pops a value from the ZSET key using WATCH/MULTI/EXEC commands.
func zpop(c redis.Conn, key string) (result string, err error) {

	defer func() {
		// Return connection to normal state on error.
		if err != nil {
			c.Do("DISCARD")
		}
	}()

	// Loop until transaction is successful.
	for {
		if _, err := c.Do("WATCH", key); err != nil {
			return "", err
		}

		members, err := redis.Strings(c.Do("ZRANGE", key, 0, 0))
		if err != nil {
			return "", err
		}
		if len(members) != 1 {
			return "", redis.ErrNil
		}

		c.Send("MULTI")
		c.Send("ZREM", key, members[0])
		queued, err := c.Do("EXEC")
		if err != nil {
			return "", err
		}

		if queued != nil {
			result = members[0]
			break
		}
	}

	return result, nil
}
func (m *baseModel) Sync(timeout time.Duration, conn redis.Conn) error {
	m.syncing.Wait()
	m.syncing.Add(1)
	defer m.syncing.Done()

	m.log.Infof("sync: Syncing %ss. Save data from cloud?:%s", m.idType, enableSyncFromCloud)

	var diffList SyncDifferenceList

	manifest, err := m.getSyncManifest(conn)
	if err != nil {
		return err
	}

	m.log.Debugf("sync: Sending %d %s local update times", len(*manifest), m.idType)

	calcClient := m.SyncConn.Conn.GetServiceClient("$ninja/services/rpc/modelstore/calculate_sync_items")
	err = calcClient.Call("modelstore.calculate_sync_items", []interface{}{m.idType, manifest}, &diffList, timeout)

	if err != nil {
		return fmt.Errorf("Failed calling calculate_sync_items for model %s error:%s", m.idType, err)
	}

	m.log.Infof("sync: Cloud requires %d %s(s), Node requires %d %s(s)", len(diffList.CloudRequires), m.idType, len(diffList.NodeRequires), m.idType)

	if len(diffList.CloudRequires)+len(diffList.NodeRequires) == 0 {
		// Nothing to do, we're in sync.
		return nil
	}

	requestIds := make([]string, 0)
	for id := range diffList.NodeRequires {
		requestIds = append(requestIds, id)
	}

	requestedData := SyncDataSet{}

	for id := range diffList.CloudRequires {
		obj := reflect.New(m.objType).Interface()

		err = m.fetch(id, obj, true, conn)

		if err != nil && err != RecordNotFound {
			return fmt.Errorf("Failed retrieving requested %s id:%s error:%s", m.idType, id, err)
		}

		if err == RecordNotFound {
			obj = nil
		}

		lastUpdated, err := m.getLastUpdated(id, conn)
		if err != nil {
			return fmt.Errorf("Failed retrieving last updated time for requested %s id:%s error:%s", m.idType, id, err)
		}

		requestedData[id] = SyncObject{obj, lastUpdated.UnixNano() / int64(time.Millisecond)}
	}

	if deleteUnknownFromCloud {
		for id := range diffList.NodeRequires {
			if _, ok := (*manifest)[id]; !ok { // We've never heard of this, so remove it
				m.log.Infof("Removing %s id:%s from cloud.", m.idType, id)
				requestedData[id] = SyncObject{nil, time.Now().UnixNano() / int64(time.Millisecond)}
			}
		}
	}

	syncClient := m.SyncConn.Conn.GetServiceClient("$ninja/services/rpc/modelstore/do_sync_items")

	var syncReply SyncReply

	err = syncClient.Call("modelstore.do_sync_items", []interface{}{m.idType, requestedData, requestIds}, &syncReply, timeout)

	if err != nil {
		return fmt.Errorf("Failed calling do_sync_items for model %s error:%s", m.idType, err)
	}

	defer syncFS()

	if enableSyncFromCloud {

		for id, requestedObj := range syncReply.RequestedObjects {
			obj := reflect.New(m.objType).Interface()

			err := json.Unmarshal(requestedObj.Data, obj)
			if err != nil {
				m.log.Warningf("Failed to unmarshal requested %s id:%s error: %s", m.idType, id, err)
				m.delete(id, conn)
			} else if string(requestedObj.Data) == "null" {
				m.log.Infof("Requested %s id:%s has been remotely deleted", m.idType, id)
				m.delete(id, conn)
			} else {

				updated, err := m.save(id, obj, conn)
				if err != nil {
					return fmt.Errorf("Failed to save requested %s id:%s error: %s", m.idType, id, err)
				}
				if !updated {
					m.log.Warningf("We requested an updated %s id:%s but it was the same as what we had.", m.idType, id)
				}

			}

			err = m.markUpdated(id, time.Unix(0, requestedObj.LastModified*int64(time.Millisecond)), conn)
			if err != nil {
				m.log.Warningf("Failed to update last modified time of requested %s id:%s error: %s", m.idType, id, err)
			}
		}
	} else {
		m.log.Warningf("Ignoring sync data from cloud.")
	}

	if err != nil {

		ts, err := time.Now().MarshalText()
		if err != nil {
			return err
		}

		_, err = conn.Do("SET", m.idType+"s:synced", ts)

	}

	return err
}