func (m *mongoStore) SaveTags(msg *common.SmapMessage) error {
if msg == nil {
return fmt.Errorf("Message is null")
}
// if the message has no metadata and is already in cache, then skip writing
if !msg.HasMetadata() && m.uuidCache.Get(string(msg.UUID)) != nil {
return nil
}
// save to the metadata database
_, err := m.metadata.Upsert(bson.M{"uuid": msg.UUID}, bson.M{"$set": msg.ToBson()})
// and save to the uuid cache
m.uuidCache.Set(string(msg.UUID), struct{}{}, m.cacheExpiry)
if msg.Properties != nil && msg.Properties.UnitOfTime != 0 {
m.uotCache.Set(string(msg.UUID), msg.Properties.UnitOfTime, m.cacheExpiry)
}
if msg.Properties != nil && msg.Properties.UnitOfMeasure != "" {
m.uomCache.Set(string(msg.UUID), msg.Properties.UnitOfMeasure, m.cacheExpiry)
}
return err
}
func (req *ArchiveRequest) GetMetadata(msg *bw.SimpleMessage) *common.SmapMessage {
var ret = new(common.SmapMessage)
req.Lock()
if req.UUID != "" && req.uuidActual == "" {
req.uuidActual = common.UUID(req.UUID)
}
req.Unlock()
ret.UUID = req.uuidActual
ret.Path = req.URI + "/" + req.Value
ret.Metadata = make(common.Dict)
ret.Properties = new(common.SmapProperties)
for _, po := range msg.POs {
var md map[string]interface{}
if po.IsTypeDF(bw.PODFMsgPack) {
err := po.(bw.MsgPackPayloadObject).ValueInto(&md)
if err != nil {
log.Error(errors.Wrap(err, "Could not unmarshal msgpack metadata"))
return nil
}
} else if po.IsTypeDF(bw.PODFSMetadata) {
md = make(map[string]interface{})
tuple := po.(bw.MetadataPayloadObject).Value()
md[getMetadataKey(msg.URI)] = tuple.Value
}
for k, v := range md {
val := fmt.Sprintf("%s", v)
if k == "UnitofTime" {
ret.Properties.UnitOfTime, _ = common.ParseUOT(val)
} else if k == "UnitofMeasure" {
ret.Properties.UnitOfMeasure = val
}
ret.Metadata[k] = val
}
}
return ret
}
// Takes an incoming common.SmapMessage object (from a client) and does the following:
// - Checks the incoming message against the ApiKey to verify it is valid to write
// - Saves the attached metadata (if any) to the metadata store
// - Reevaluates any dynamic subscriptions and pushes to republish clients
// - Saves the attached readings (if any) to the timeseries database
func (a *Archiver) AddData(msg *common.SmapMessage) (err error) {
// save metadata
err = a.mdStore.SaveTags(msg)
if err != nil {
return err
}
// fix inconsistencies
var (
uot common.UnitOfTime
uom string
)
if uot, err = a.mdStore.GetUnitOfTime(msg.UUID); uot == 0 && err == nil {
if len(msg.Readings) > 0 {
uot = common.GuessTimeUnit(msg.Readings[0].GetTime())
}
} else if err != nil {
return err
}
for _, rdg := range msg.Readings {
rdg.SetUOT(uot)
}
if uom, err = a.mdStore.GetUnitOfMeasure(msg.UUID); uom == "" && err == nil {
if msg.Properties == nil {
msg.Properties = &common.SmapProperties{StreamType: common.NUMERIC_STREAM}
}
msg.Properties.UnitOfMeasure = "n/a"
err = a.mdStore.SaveTags(msg)
if err != nil {
return err
}
} else if err != nil {
return err
}
//save timeseries data
a.metrics["adds"].Mark(1)
a.tsStore.AddMessage(msg)
a.broker.HandleMessage(msg)
return err
}
func (req *ArchiveRequest) GetSmapMessage(thing interface{}) *common.SmapMessage {
var msg = new(common.SmapMessage)
var rdg = new(common.SmapNumberReading)
value := ob.Eval(req.value, thing)
switch t := value.(type) {
case int64:
rdg.Value = float64(t)
case uint64:
rdg.Value = float64(t)
case float64:
rdg.Value = t
}
rdg.Time = req.getTime(thing)
if len(req.uuid) > 0 && req.uuidActual == "" {
req.uuidActual = common.UUID(ob.Eval(req.uuid, thing).(string))
} else if req.uuidActual == "" {
req.uuidActual = common.UUID(req.UUID)
}
msg.UUID = req.uuidActual
msg.Path = req.URI + "/" + req.Value
msg.Readings = []common.Reading{rdg}
if len(req.metadataExpr) > 0 {
msg.Metadata = make(common.Dict)
msg.Properties = new(common.SmapProperties)
if md, ok := ob.Eval(req.metadataExpr, thing).(map[string]interface{}); ok {
for k, v := range md {
val := fmt.Sprintf("%s", v)
if k == "UnitofTime" {
msg.Properties.UnitOfTime, _ = common.ParseUOT(val)
} else if k == "UnitofMeasure" {
msg.Properties.UnitOfMeasure = val
}
msg.Metadata[k] = val
}
}
}
return msg
}
// First, we check that all the fields are valid and the necessary ones are populated.
// This also involves filling in the optional ones with sane values.
// Then we build a chain on the URI to the VK -- if this fails, then we stop
// Then we build the operator chains for the expressions required
// Then we subscribe to the URI indicated.
func (bwh *BOSSWaveHandler) ParseArchiveRequest(request *ArchiveRequest) (*URIArchiver, error) {
if request.FromVK == "" {
return nil, errors.New("VK was empty in ArchiveRequest")
}
request.value = ob.Parse(request.Value)
if request.UUID == "" {
request.UUID = uuid.NewV3(NAMESPACE_UUID, request.URI+string(request.PO)+request.Value).String()
} else {
request.uuid = ob.Parse(request.UUID)
}
if request.Time != "" {
request.time = ob.Parse(request.Time)
}
if request.MetadataExpr != "" {
request.metadataExpr = ob.Parse(request.MetadataExpr)
}
if request.InheritMetadata {
md, _, err := bwh.bw.GetMetadata(request.URI)
if err != nil {
return nil, err
}
var ret = new(common.SmapMessage)
request.Lock()
if request.UUID != "" && request.uuidActual == "" {
request.uuidActual = common.UUID(request.UUID)
}
request.Unlock()
ret.UUID = request.uuidActual
ret.Path = request.URI + "/" + request.Value
ret.Metadata = make(common.Dict)
ret.Properties = new(common.SmapProperties)
for k, v := range md {
val := fmt.Sprintf("%s", v.Value)
if k == "UnitofTime" {
ret.Properties.UnitOfTime, _ = common.ParseUOT(val)
} else if k == "UnitofMeasure" {
ret.Properties.UnitOfMeasure = val
}
ret.Metadata[k] = val
}
if err = bwh.a.AddData(ret); err != nil {
log.Error(errors.Wrap(err, "Could not add data"))
}
}
var metadataChan = make(chan *bw.SimpleMessage)
if len(request.MetadataURIs) > 0 {
for _, metadataURI := range request.MetadataURIs {
sub1, err := bwh.bw.Subscribe(&bw.SubscribeParams{
URI: strings.TrimSuffix(metadataURI, "/") + "/!meta/+",
})
if err != nil {
return nil, err
}
go func() {
for msg := range sub1 {
metadataChan <- msg
}
}()
q1, err := bwh.bw.Query(&bw.QueryParams{
URI: strings.TrimSuffix(metadataURI, "/") + "/!meta/+",
})
if err != nil {
return nil, err
}
go func() {
for msg := range q1 {
metadataChan <- msg
}
}()
}
}
//TODO: subscribe then query MetadataBlock
log.Debugf("Subscribing for Archival on %s", request.URI)
sub, err := bwh.bw.Subscribe(&bw.SubscribeParams{
URI: request.URI,
})
if err != nil {
return nil, errors.Wrap(err, "Could not subscribe")
}
log.Debugf("Got archive request")
request.Dump()
archiver := &URIArchiver{sub, metadataChan, request}
go archiver.Listen(bwh.a)
return archiver, nil
}