func getAnswerKey(db *mgo.Database, recMatchRun *ptm_models.RecordMatchRun) (*fhir_models.Bundle, error) {
var answerKey *fhir_models.Bundle
// If deduplication mode, try to find an answer key w/ the master record set
if recMatchRun.MatchingMode == ptm_models.Deduplication {
masterRecSet := &ptm_models.RecordSet{}
c := db.C(ptm_models.GetCollectionName("RecordSet"))
// retrieve the master record set
err := c.Find(
bson.M{"_id": recMatchRun.MasterRecordSetID}).One(masterRecSet)
if err != nil {
logger.Log.WithFields(logrus.Fields{"func": "updateRecordMatchRun",
"err": err,
"msg": "Unable to find master record set",
"record setid": recMatchRun.MasterRecordSetID}).Warn("calcMetrics")
return nil, err
}
logger.Log.WithFields(logrus.Fields{
"rec set": masterRecSet,
"answer key entries": len(masterRecSet.AnswerKey.Entry)}).Info("calcMetrics")
if len(masterRecSet.AnswerKey.Entry) > 1 {
answerKey = &masterRecSet.AnswerKey
logger.Log.WithFields(logrus.Fields{
"answer key entries": len(masterRecSet.AnswerKey.Entry)}).Info("calcMetrics")
}
} else if recMatchRun.MatchingMode == ptm_models.Query {
logger.Log.WithFields(logrus.Fields{
"msg": "Calculating Metrics for Query Mode Not Supoprted"}).Warn("calcMetrics")
return nil, nil
}
return answerKey, nil
}
func (rc *ResourceController) GetResources(ctx *gin.Context) {
req := ctx.Request
resourceType := getResourceType(req.URL)
logger.Log.WithFields(
logrus.Fields{"resource type": resourceType}).Info("GetResources")
resources := ptm_models.NewSliceForResourceName(resourceType, 0, 0)
c := rc.Database().C(ptm_models.GetCollectionName(resourceType))
// retrieve all documents in the collection
// TODO Restrict this to resource type, just to be extra safe
query := buildSearchQuery(resourceType, ctx)
logger.Log.WithFields(
logrus.Fields{"query": query}).Info("GetResources")
err := c.Find(query).All(resources)
if err != nil {
if err == mgo.ErrNotFound {
ctx.String(http.StatusNotFound, "Not Found")
ctx.Abort()
return
} else {
ctx.AbortWithError(http.StatusBadRequest, err)
return
}
}
ctx.JSON(http.StatusOK, resources)
}
func GetRecordMatchRunLinksHandler(provider func() *mgo.Database) gin.HandlerFunc {
return func(ctx *gin.Context) {
idString := ctx.Param("id")
id := bson.ObjectIdHex(idString)
c := provider().C(ptm_models.GetCollectionName("RecordMatchRun"))
rmr := &ptm_models.RecordMatchRun{}
err := c.Find(bson.M{"_id": id}).One(rmr)
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, err)
return
}
category := ctx.Query("category")
limitString := ctx.Query("limit")
limit, err := strconv.ParseInt(limitString, 10, 0)
if err != nil || limit == 0 {
limit = 10
}
var links []ptm_models.Link
if category == "worst" {
links = rmr.GetWorstLinks(int(limit))
} else {
links = rmr.GetBestLinks(int(limit))
}
ctx.JSON(http.StatusOK, links)
}
}
func GetRecordMatchRunMetricsHandler(provider func() *mgo.Database) gin.HandlerFunc {
return func(ctx *gin.Context) {
resourceType := "RecordMatchRun"
recordMatchSystemInterfaceId := ctx.Query("recordMatchSystemInterfaceId")
validRecordMatchSystemInterfaceId := len(recordMatchSystemInterfaceId) > 1 && len(recordMatchSystemInterfaceId) <= 24 && bson.IsObjectIdHex(recordMatchSystemInterfaceId)
recordSetId := ctx.Query("recordSetId")
validRecordSetId := len(recordSetId) > 1 && len(recordSetId) <= 24 && bson.IsObjectIdHex(recordSetId)
logger.Log.WithFields(
logrus.Fields{"resource type": resourceType,
"rec match sys": recordMatchSystemInterfaceId,
"record set": recordSetId}).Info("GetRecordMatchRunMetrics")
resources := ptm_models.NewSliceForResourceName(resourceType, 0, 0)
c := provider().C(ptm_models.GetCollectionName(resourceType))
var query *mgo.Query
if validRecordSetId {
logger.Log.WithFields(
// find the record match runs with masterRecordSetId or queryRecordSetId == record set id
logrus.Fields{"validRecord Set Id": validRecordSetId, "record set": recordSetId}).Info("GetRecordMatchRunMetrics")
recordSetBsonID, _ := ptm_models.ToBsonObjectID(recordSetId)
query = c.Find(bson.M{"$or": []bson.M{bson.M{"masterRecordSetId": recordSetBsonID}, bson.M{"queryRecordSetId": recordSetBsonID}}})
} else if validRecordMatchSystemInterfaceId {
recordMatchSystemInterfaceBsonId, _ := ptm_models.ToBsonObjectID(recordMatchSystemInterfaceId)
query = c.Find(bson.M{"recordMatchSystemInterfaceId": recordMatchSystemInterfaceBsonId})
} else { // no query parameters were provided
// get all record runs with, primarily, metrics only
// retrieve all documents in the collection
// TODO Restrict this to resourc type, just to be extra safe
query = c.Find(bson.M{})
}
// constrain which fields are returned
err := query.Select(bson.M{"meta": 1, "metrics": 1,
"recordMatchSystemInterfaceId": 1, "matchingMode": 1,
"recordResourceType": 1, "masterRecordSetId": 1, "queryRecordSetId": 1,
"recordMatchContextId": 1}).All(resources)
if err != nil {
if err == mgo.ErrNotFound {
ctx.String(http.StatusNotFound, "Not Found")
ctx.Abort()
return
}
ctx.AbortWithError(http.StatusBadRequest, err)
}
ctx.JSON(http.StatusOK, resources)
}
}
// LoadResource returns an object from the database that matches the specified
// resource type and object identifier.
func (rc *ResourceController) LoadResource(resourceType string, id bson.ObjectId) (interface{}, error) {
// Determine the collection expected to hold the resource
c := rc.Database().C(ptm_models.GetCollectionName(resourceType))
result := ptm_models.NewStructForResourceName(resourceType)
err := c.Find(bson.M{"_id": id}).One(result)
if err != nil {
return nil, err
}
logger.Log.WithFields(logrus.Fields{"result": result}).Debug("LoadResource")
return result, nil
}
// DeleteResource handles requests to delete a specific resource.
func (rc *ResourceController) DeleteResource(ctx *gin.Context) {
var id bson.ObjectId
req := ctx.Request
resourceType := getResourceType(req.URL)
// Validate id as a bson Object ID
id, err := toBsonObjectID(ctx.Param("id"))
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, err)
return
}
logger.Log.WithFields(
logrus.Fields{"resource type": resourceType, "id": id, "coll": ptm_models.GetCollectionName(resourceType)}).Info("DeleteResource")
// Determine the collection expected to hold the resource
c := rc.Database().C(ptm_models.GetCollectionName(resourceType))
err = c.Remove(bson.M{"_id": id})
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, err)
return
}
ctx.Status(http.StatusNoContent)
}
// SetAnswerKey associates a specified Record Set with a FHIR Bundle that
// contains a set of expected record matches (i.e., answer key for the record set)
// The uploaded file is expected to be a FHIR Bundle of type, document,
// in JSON representation.
func (rc *ResourceController) SetAnswerKey(ctx *gin.Context) {
recordSetId, err := toBsonObjectID(ctx.PostForm("recordSetId"))
// Ensure the referenced Record Set exists
resource, err := rc.LoadResource("RecordSet", recordSetId)
if err != nil {
ctx.AbortWithError(http.StatusNotFound, err)
return
}
recordSet := resource.(*ptm_models.RecordSet)
// extract the answer key from the posted form
file, _, err := ctx.Request.FormFile("answerKey")
if err != nil {
ctx.AbortWithError(http.StatusBadRequest, err)
return
}
// write uploaded content to a temp file
tmpfile, err := ioutil.TempFile(os.TempDir(), "ptmatch-")
defer os.Remove(tmpfile.Name())
_, err = io.Copy(tmpfile, file)
ptm_models.LoadResourceFromFile(tmpfile.Name(), &recordSet.AnswerKey)
if isValidAnswerKey(recordSet.AnswerKey) {
c := rc.Database().C(ptm_models.GetCollectionName("RecordSet"))
err = c.Update(bson.M{"_id": recordSetId}, recordSet)
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, err)
return
}
resource, err = rc.LoadResource("RecordSet", recordSetId)
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, err)
return
}
recordSet = resource.(*ptm_models.RecordSet)
logger.Log.WithFields(
logrus.Fields{"updated recordset": recordSet}).Info("SetAnswerKey")
ctx.JSON(http.StatusOK, recordSet)
} else {
ctx.AbortWithStatus(http.StatusBadRequest)
}
}
func (rc *ResourceController) UpdateResource(ctx *gin.Context) {
var id bson.ObjectId
// Section 9.6 of RFC 2616 says to return 201 if resource didn't already exist
// and 200 or 204, otherwise
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html#sec9.6
var statusCode int = http.StatusOK
req := ctx.Request
resourceType := getResourceType(req.URL)
// Validate id as a bson Object ID
id, err := toBsonObjectID(ctx.Param("id"))
if err != nil {
ctx.AbortWithError(http.StatusBadRequest, err)
return
}
var createdOn reflect.Value
// Determine if the resource already exists
existing, err := rc.LoadResource(resourceType, id)
if err != nil {
if err == mgo.ErrNotFound {
statusCode = http.StatusCreated
} else {
ctx.AbortWithError(http.StatusInternalServerError, err)
return
}
} else {
// reflect.ValueOf(&n).Elem().FieldByName("N").Set(reflect.ValueOf(ft))
metaField := reflect.ValueOf(existing).Elem().FieldByName("Meta")
createdOn = metaField.Elem().FieldByName("CreatedOn")
logger.Log.WithFields(
logrus.Fields{"createdOn": createdOn}).Info("UpdateResource")
}
resource := ptm_models.NewStructForResourceName(resourceType)
if err := ctx.Bind(resource); err != nil {
ctx.AbortWithError(http.StatusInternalServerError, err)
return
}
c := rc.Database().C(ptm_models.GetCollectionName(resourceType))
// Force the ID provided in the URL to be in the resource object
reflect.ValueOf(resource).Elem().FieldByName("ID").Set(reflect.ValueOf(id))
ptm_models.UpdateLastUpdatedDate(resource)
// Ensure the creation date does not change`
metaField := reflect.ValueOf(resource).Elem().FieldByName("Meta")
metaField.Elem().FieldByName("CreatedOn").Set(createdOn)
createdOn2 := metaField.Elem().FieldByName("CreatedOn")
logger.Log.WithFields(
logrus.Fields{"createdOn2": createdOn2}).Info("UpdateResource")
err = c.Update(bson.M{"_id": id}, resource)
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, err)
return
}
logger.Log.WithFields(
logrus.Fields{"collection": ptm_models.GetCollectionName(resourceType),
"res type": resourceType, "id": id, "createdOn": createdOn}).Info("UpdateResource")
ctx.Header("Location", responseURL(req, resourceType, id.Hex()).String())
ctx.JSON(statusCode, resource)
}
func calcMetrics(db *mgo.Database, recMatchRun *ptm_models.RecordMatchRun,
respMsg *fhir_models.Bundle) error {
answerKey, _ := getAnswerKey(db, recMatchRun)
numAnswers := 0
var answerMap map[string]*groundTruth
if answerKey != nil {
// store answer key info in map
answerMap, numAnswers = buildAnswerMap(answerKey)
}
metrics := recMatchRun.Metrics
logger.Log.WithFields(logrus.Fields{
"metrics": metrics}).Info("calcMetrics")
//expectedResourceType := recMatchRun.RecordResourceType
matchCount := 0
truePositiveCount := 0
falsePositiveCount := 0
for i, entry := range respMsg.Entry {
// logger.Log.WithFields(logrus.Fields{
// "i": i,
// "entry type": rtype,
// "entry kind": reflect.ValueOf(entry.Resource).Kind()}).Info("calcMetrics")
// Results are in untyped entry w/ links and search result
if entry.Resource == nil {
refURL := entry.FullUrl
if refURL != "" && entry.Search != nil && len(entry.Link) > 0 {
score := *entry.Search.Score
// logger.Log.WithFields(logrus.Fields{
// "i": i,
// "full url": refURL,
// "search": score}).Info("calcMetrics")
if score > 0 {
for _, link := range entry.Link {
if strings.EqualFold("related", link.Relation) {
linkedURL := link.Url
matchCount++
logger.Log.WithFields(logrus.Fields{
"i": i,
"full url": refURL,
"link url": linkedURL,
"search": score}).Info("calcMetrics")
// if we have an answer key to compare against
if answerKey != nil {
truth := answerMap[refURL]
if truth != nil {
// look for linked URL in array of known linked records
idx := indexOf(truth.linkedURLs, linkedURL)
if idx >= 0 {
truePositiveCount++
truth.numFound[idx]++
} else {
falsePositiveCount++
}
} else if answerMap[linkedURL] != nil {
truth := answerMap[linkedURL]
// look for reference URL in array of known linked records
idx := indexOf(truth.linkedURLs, refURL)
if idx >= 0 {
truePositiveCount++
truth.numFound[idx]++
} else {
falsePositiveCount++
}
} else {
// no entry found in answer key; this is a false positive
falsePositiveCount++
}
}
}
}
}
}
}
}
logger.Log.WithFields(logrus.Fields{
"truePositive": truePositiveCount,
"falsePositive": falsePositiveCount,
"matchCount": matchCount}).Info("calcMetrics")
metrics.MatchCount += matchCount
// if there is an answer key w/ answers and some results were processed
if answerKey != nil && numAnswers > 0 && matchCount > 0 {
metrics.TruePositiveCount += truePositiveCount
metrics.FalsePositiveCount += falsePositiveCount
metrics.Precision = float32(metrics.TruePositiveCount) / float32(metrics.TruePositiveCount+metrics.FalsePositiveCount)
metrics.Recall = float32(metrics.TruePositiveCount) / float32(numAnswers)
metrics.F1 = 2.0 * ((metrics.Precision * metrics.Recall) / (metrics.Precision + metrics.Recall))
}
now := time.Now()
c := db.C(ptm_models.GetCollectionName("RecordMatchRun"))
// Add an entry to the record match run status and update lastUpdatedOn
err := c.UpdateId(recMatchRun.ID,
bson.M{
"$currentDate": bson.M{"meta.lastUpdatedOn": bson.M{"$type": "timestamp"}},
"$set": bson.M{"metrics": metrics},
"$push": bson.M{
"status": bson.M{
"message": "Metrics Updated [" + respMsg.Id + "]",
"createdOn": now}}})
if err != nil {
logger.Log.WithFields(logrus.Fields{"msg": "Error updating metrics in record match run",
"rec match run ID": recMatchRun.ID,
"error": err}).Warn("calcMetrics")
return err
}
return nil
}
func updateRecordMatchRun(db *mgo.Database, respMsg *fhir_models.Bundle) error {
// Verify this bundle represents a message
if respMsg.Type == "message" {
// we care only about response messages
msgHdr := respMsg.Entry[0].Resource.(*fhir_models.MessageHeader)
resp := msgHdr.Response
logger.Log.WithFields(logrus.Fields{"action": "Recognized Bundle of type, message",
"bundle id": respMsg.Id,
"msg hdr": msgHdr}).Info("updateRecordMatchRun")
// verify this is a response for a record-match request
if resp != nil &&
msgHdr.Event.Code == "record-match" &&
msgHdr.Event.System == "http://github.com/mitre/ptmatch/fhir/message-events" {
reqID := resp.Identifier
// Determine the collection expected to hold the resource
c := db.C(ptm_models.GetCollectionName("RecordMatchRun"))
recMatchRun := &ptm_models.RecordMatchRun{}
// retrieve the record-match run
err := c.Find(
bson.M{"request.message.entry.resource._id": reqID}).One(recMatchRun)
if err != nil {
logger.Log.WithFields(logrus.Fields{"func": "updateRecordMatchRun",
"err": err,
"request msg id": reqID}).Warn("Unable to find RecMatchRun assoc w. request")
return err
}
logger.Log.WithFields(logrus.Fields{"action": "found run assoc w. request",
"result": recMatchRun}).Info("updateRecordMatchRun")
now := time.Now()
// check whether the response is already assoc. w/ the record match run object
count, err := c.Find(bson.M{"_id": recMatchRun.ID,
"responses.message._id": respMsg.Id}).Count()
logger.Log.WithFields(logrus.Fields{"action": "look for dupl response",
"respMsg.Id": respMsg.Id,
"count": count}).Info("updateRecordMatchRun")
if count > 0 {
// The response message has been processed before
logger.Log.WithFields(logrus.Fields{"action": "record match response seen before",
"record match run": recMatchRun.ID,
"response msg Id": respMsg.Id}).Info("updateRecordMatchRun")
// Record that we've seen this response before
err = c.UpdateId(recMatchRun.ID,
bson.M{
"$currentDate": bson.M{"meta.lastUpdatedOn": bson.M{"$type": "timestamp"}},
"$push": bson.M{
"status": bson.M{
"message": "Duplicate Response Received and Ignored [" + respMsg.Id + "]",
"createdOn": now}}})
return nil
}
var respID bson.ObjectId
// if the bundle id looks like a bson object id, use it; else we need
// to create a bson id 'cuz IE fhir server only supports those (5/10/16)'
if bson.IsObjectIdHex(respMsg.Id) {
respID = bson.ObjectIdHex(respMsg.Id)
} else {
logger.Log.WithFields(logrus.Fields{"msg": "Response Msg Id is not BSON Object Id format",
"rec match run ID": recMatchRun.ID,
"respMsg.id": respMsg.Id}).Warn("updateRecordMatchRun")
respID = bson.NewObjectId()
}
// Add the record match response to the record run data
err = c.UpdateId(recMatchRun.ID,
bson.M{"$push": bson.M{"responses": bson.M{
"_id": respID,
"meta": bson.M{"lastUpdatedOn": now, "createdOn": now},
"receivedOn": now,
"message": respMsg,
}}})
if err != nil {
logger.Log.WithFields(logrus.Fields{"msg": "Error adding response to Run Info",
"rec match run ID": recMatchRun.ID,
"error": err}).Warn("eupdateRecordMatchRun")
return err
}
// Add an entry to the record match run status and update lastUpdatedOn
err = c.UpdateId(recMatchRun.ID,
bson.M{
"$currentDate": bson.M{"meta.lastUpdatedOn": bson.M{"$type": "timestamp"}},
"$push": bson.M{
"status": bson.M{
"message": "Response Received [" + respMsg.Id + "]",
"createdOn": now}}})
if err != nil {
logger.Log.WithFields(logrus.Fields{"msg": "Error updating response status in run object",
"rec match run ID": recMatchRun.ID,
"error": err}).Warn("updateRecordMatchRun")
return err
}
// Calculate metrics
_ = calcMetrics(db, recMatchRun, respMsg)
}
}
return nil
}
