// Fetch historic time series data for a cloud variable. The resolution is
// automatically selected.
func (device *CassDevice) HistoricData(
varDef sddl.VarDef,
curTime,
startTime,
endTime time.Time) ([]cloudvar.CloudVarSample, error) {
canolog.Info("Fetching historic data for", varDef.Name(), startTime, endTime)
// Figure out which resolution to use.
// Pick the highest resolution that covers the entire requested period.
var lod lodEnum
for lod = LOD_0; lod < LOD_END; lod++ {
lodDuration := cloudVarLODDuration(TIER_STANDARD, lod)
// TODO: Should we use curTime or lastUpdateTime for this?
if startTime.After(curTime.Add(-lodDuration)) {
break
}
}
if lod == LOD_END {
lod = LOD_5
}
canolog.Info("Using LOD", lod)
// Fetch the data from that LOD
return device.historicDataLOD(varDef, startTime, endTime, lod)
}
// Insert a sample into the database for a particular LOD level, discarding the
// sample if the stratification chunk already contains a sample.
func (device *CassDevice) insertOrDiscardSampleLOD(varDef sddl.VarDef,
lastUpdateTime time.Time,
lod lodEnum,
t time.Time,
value interface{}) error {
// Discard sample if it doesn't cross a stratification boundary
stratificationSize := lodStratificationSize[lod]
if !crossesStratificationBoundary(lastUpdateTime, t, stratificationSize) {
// discard sample
canolog.Info("LOD", lod, "discarded")
return nil
}
// Get table name
tableName, err := varTableNameByDatatype(varDef.Datatype())
if err != nil {
return err
}
// insert sample
bucket := getBucket(t, lod)
propname := varDef.Name()
err = device.conn.session.Query(`
INSERT INTO `+tableName+`
(device_id, propname, timeprefix, time, value)
VALUES (?, ?, ?, ?, ?)
`, device.ID(), propname, bucket.Name(), t, value).Exec()
if err != nil {
return err
}
canolog.Info("LOD", lod, "sample inserted into bucket", bucket.Name())
// Track new bucket (if any) for garbage collection purposes.
// And garbage collect.
if crossesBucketBoundary(lastUpdateTime, t, bucket.BucketSize()) {
err := device.addBucket(propname, &bucket)
canolog.Info("New bucket", bucket, "created")
if err != nil {
canolog.Error("Error adding sample bucket: ", err)
// don't return! We still need to do garbage collection!
}
device.garbageCollectLOD(t, varDef, lod, false)
}
return nil
}
// Insert a cloud variable data sample.
func (device *CassDevice) InsertSample(varDef sddl.VarDef, t time.Time, value interface{}) error {
// Convert to UTC before inserting
t = t.UTC()
canolog.Info("Inserting sample", varDef.Name(), t)
// check last update time
lastUpdateTime, err := device.varLastUpdateTime(varDef.Name())
if err != nil {
canolog.Error("Error inserting sample:", err.Error())
return err
}
canolog.Info("Last update time was", lastUpdateTime)
if t.Before(lastUpdateTime) {
canolog.Error("Insertion time before last update time: ", t, lastUpdateTime)
return fmt.Errorf("Insertion time %s before last update time %s", t, lastUpdateTime)
}
// update last update time
err = device.varSetLastUpdateTime(varDef.Name(), t)
if err != nil {
return err
}
// For each LOD, insert or discard sample based on our
// stratification algorithm.
for lod := LOD_0; lod < LOD_END; lod++ {
err = device.insertOrDiscardSampleLOD(varDef, lastUpdateTime, lod, t, value)
if err != nil {
// TODO: Transactionize/rollback?
return err
}
}
// TODO: Do we need to update in-memory device object?
return nil
}
func JsonToCloudVarValue(varDef sddl.VarDef, value interface{}) (interface{}, error) {
switch varDef.Datatype() {
case sddl.DATATYPE_VOID:
return nil, nil
case sddl.DATATYPE_STRING:
v, ok := value.(string)
if !ok {
return nil, fmt.Errorf("JsonToCloudVarValue expects string value for %s", varDef.Name())
}
return v, nil
case sddl.DATATYPE_BOOL:
v, ok := value.(bool)
if !ok {
return nil, fmt.Errorf("JsonToCloudVarValue expects bool value for %s", varDef.Name())
}
return v, nil
case sddl.DATATYPE_INT8:
v, ok := value.(float64)
if !ok {
return nil, fmt.Errorf("JsonToCloudVarValue expects number value for %s", varDef.Name())
}
return int8(v), nil
case sddl.DATATYPE_UINT8:
v, ok := value.(float64)
if !ok {
return nil, fmt.Errorf("JsonToCloudVarValue expects number value for %s", varDef.Name())
}
return uint8(v), nil
case sddl.DATATYPE_INT16:
v, ok := value.(float64)
if !ok {
return nil, fmt.Errorf("JsonToCloudVarValue expects number value for %s", varDef.Name())
}
return int16(v), nil
case sddl.DATATYPE_UINT16:
v, ok := value.(float64)
if !ok {
return nil, fmt.Errorf("JsonToCloudVarValue expects number value for %s", varDef.Name())
}
return uint16(v), nil
case sddl.DATATYPE_INT32:
v, ok := value.(float64)
if !ok {
return nil, fmt.Errorf("JsonToCloudVarValue expects number value for %s", varDef.Name())
}
return int32(v), nil
case sddl.DATATYPE_UINT32:
v, ok := value.(float64)
if !ok {
return nil, fmt.Errorf("JsonToCloudVarValue expects number value for %s", varDef.Name())
}
return uint32(v), nil
case sddl.DATATYPE_FLOAT32:
v, ok := value.(float64)
if !ok {
return nil, fmt.Errorf("JsonToCloudVarValue expects number value for %s", varDef.Name())
}
return float32(v), nil
case sddl.DATATYPE_FLOAT64:
v, ok := value.(float64)
if !ok {
return nil, fmt.Errorf("JsonToCloudVarValue expects number value for %s", varDef.Name())
}
return v, nil
case sddl.DATATYPE_DATETIME:
v, ok := value.(string)
if !ok {
return nil, fmt.Errorf("JsonToCloudVarValue expects string value for %s", varDef.Name())
}
tval, err := time.Parse(time.RFC3339, v)
if err != nil {
return nil, fmt.Errorf("JsonToCloudVarValue expects RFC3339 formatted time value for %s", varDef.Name())
}
return tval, nil
default:
return nil, fmt.Errorf("InsertSample unsupported datatype ", varDef.Datatype())
}
}
func (device *CassDevice) LatestData(varDef sddl.VarDef) (*cloudvar.CloudVarSample, error) {
return device.getLatestData_generic(varDef.Name(), varDef.Datatype())
}
// Remove old buckets for a single cloud variable and LOD
// Set <deleteAll> to false for normal garbage collection (only expired buckets
// are removed). Set <deleteAll> to true to delete all data, expired or not.
func (device *CassDevice) garbageCollectLOD(curTime time.Time,
varDef sddl.VarDef,
lod lodEnum,
deleteAll bool) error {
canolog.Info("Running garbage collection for ", varDef.Name(), "LOD", lod)
// Get list of expired buckets for that LOD
var bucketName string
bucketsToRemove := []string{}
query := device.conn.session.Query(`
SELECT timeprefix, endtime
FROM var_buckets
WHERE device_id = ?
AND var_name = ?
AND lod = ?
ORDER BY timeprefix DESC
`, device.ID(), varDef.Name(), lod).Consistency(gocql.One)
iter := query.Iter()
var endTime time.Time
// NOTE: As a special case, we never delete the most recent LOD0 bucket,
// even if it has expired, because we need it for LastUpdateTime.
skipFirst := (lod == LOD_0)
for iter.Scan(&bucketName, &endTime) {
// determine expiration time
// TODO: Handle tiers
if deleteAll || bucketExpired(curTime, endTime, TIER_STANDARD, lod) {
if skipFirst {
skipFirst = false
} else {
bucketsToRemove = append(bucketsToRemove, bucketName)
}
}
}
err := iter.Close()
if err != nil {
return fmt.Errorf("Error garbage collecting cloudvar: %s", err.Error())
}
// Remove buckets
for _, bucketName := range bucketsToRemove {
// Get table name
tableName, err := varTableNameByDatatype(varDef.Datatype())
if err != nil {
return err
}
// Remove expired bucket
canolog.Info("Removing expired bucket", varDef.Name(), bucketName)
err = device.conn.session.Query(`
DELETE FROM `+tableName+`
WHERE device_id = ?
AND propname = ?
AND timeprefix = ?
`, device.ID(), varDef.Name(), bucketName).Consistency(gocql.One).Exec()
if err != nil {
canolog.Error("Problem deleting bucket ", device.ID(), varDef.Name(), bucketName)
} else {
// Cleanup var_buckets table, but only if we actually deleted the
// bucket in the previous step
err := device.conn.session.Query(`
DELETE FROM var_buckets
WHERE device_id = ?
AND var_name = ?
AND lod = ?
AND timeprefix = ?
`, device.ID(), varDef.Name(), lod, bucketName).Consistency(gocql.One).Exec()
if err != nil {
canolog.Error("Problem cleaning var_buckets ", device.ID(), varDef.Name(), bucketName, ":", err)
}
}
}
return nil
}
// Append the samples in bucket <bucketName> that fall between <startTime> and
// <endTime> to <apendee>
func (device *CassDevice) fetchAndAppendBucketSamples(varDef sddl.VarDef,
apendee []cloudvar.CloudVarSample,
startTime,
endTime time.Time,
bucketName string) ([]cloudvar.CloudVarSample, error) {
// Get table name
tableName, err := varTableNameByDatatype(varDef.Datatype())
if err != nil {
return []cloudvar.CloudVarSample{}, err
}
query := device.conn.session.Query(`
SELECT time, value
FROM `+tableName+`
WHERE device_id = ?
AND propname = ?
AND timeprefix = ?
AND time >= ?
AND time <= ?
`, device.ID(), varDef.Name(), bucketName, startTime, endTime).Consistency(gocql.One)
iter := query.Iter()
var timestamp time.Time
switch varDef.Datatype() {
case sddl.DATATYPE_VOID:
var value interface{}
for iter.Scan(×tamp) {
apendee = append(apendee, cloudvar.CloudVarSample{timestamp, value})
}
case sddl.DATATYPE_STRING:
var value string
for iter.Scan(×tamp, &value) {
apendee = append(apendee, cloudvar.CloudVarSample{timestamp, value})
}
case sddl.DATATYPE_BOOL:
var value bool
for iter.Scan(×tamp, &value) {
apendee = append(apendee, cloudvar.CloudVarSample{timestamp, value})
}
case sddl.DATATYPE_INT8:
var value int8
for iter.Scan(×tamp, &value) {
apendee = append(apendee, cloudvar.CloudVarSample{timestamp, value})
}
case sddl.DATATYPE_UINT8:
var value uint8
for iter.Scan(×tamp, &value) {
apendee = append(apendee, cloudvar.CloudVarSample{timestamp, value})
}
case sddl.DATATYPE_INT16:
var value int16
for iter.Scan(×tamp, &value) {
apendee = append(apendee, cloudvar.CloudVarSample{timestamp, value})
}
case sddl.DATATYPE_UINT16:
var value uint16
for iter.Scan(×tamp, &value) {
apendee = append(apendee, cloudvar.CloudVarSample{timestamp, value})
}
case sddl.DATATYPE_INT32:
var value int32
for iter.Scan(×tamp, &value) {
apendee = append(apendee, cloudvar.CloudVarSample{timestamp, value})
}
case sddl.DATATYPE_UINT32:
var value uint32
for iter.Scan(×tamp, &value) {
apendee = append(apendee, cloudvar.CloudVarSample{timestamp, value})
}
case sddl.DATATYPE_FLOAT32:
var value float32
for iter.Scan(×tamp, &value) {
apendee = append(apendee, cloudvar.CloudVarSample{timestamp, value})
}
case sddl.DATATYPE_FLOAT64:
var value float64
for iter.Scan(×tamp, &value) {
apendee = append(apendee, cloudvar.CloudVarSample{timestamp, value})
}
case sddl.DATATYPE_DATETIME:
var value time.Time
for iter.Scan(×tamp, &value) {
apendee = append(apendee, cloudvar.CloudVarSample{timestamp, value})
}
case sddl.DATATYPE_INVALID:
return []cloudvar.CloudVarSample{}, fmt.Errorf("Cannot get property values for DATATYPE_INVALID")
default:
return []cloudvar.CloudVarSample{}, fmt.Errorf("Cannot get property values for datatype %d", varDef.Datatype())
}
err = iter.Close()
if err != nil {
return []cloudvar.CloudVarSample{}, err
}
return apendee, nil
}