func (cdre *CdrExporter) getDateTimeFieldVal(cdr *engine.StoredCdr, cfgCdrFld *config.CfgCdrField) (string, error) {
if len(cfgCdrFld.Value) == 0 {
return "", nil
}
for _, fltrRl := range cfgCdrFld.FieldFilter {
if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass {
return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl)
}
}
layout := cfgCdrFld.Layout
if len(layout) == 0 {
layout = time.RFC3339
}
if dtFld, err := utils.ParseTimeDetectLayout(cdr.FieldAsString(cfgCdrFld.Value[0]), cdre.timezone); err != nil { // Only one rule makes sense here
return "", err
} else {
return dtFld.Format(layout), nil
}
}
func (cdre *CdrExporter) getCombimedCdrFieldVal(processedCdr *engine.StoredCdr, cfgCdrFld *config.CfgCdrField) (string, error) {
var combinedVal string // Will result as combination of the field values, filters must match
for _, filterRule := range cfgCdrFld.FieldFilter {
fltrPass, ftrPassValue := processedCdr.PassesFieldFilter(filterRule)
if !fltrPass {
return "", nil
}
for _, cdr := range cdre.cdrs {
if cdr.CgrId != processedCdr.CgrId {
continue // We only care about cdrs with same primary cdr behind
}
if cdr.FieldAsString(&utils.RSRField{Id: filterRule.Id}) == ftrPassValue { // First CDR with filte
for _, rsrRule := range cfgCdrFld.Value {
combinedVal += cdr.FieldAsString(rsrRule)
}
}
}
}
return combinedVal, nil
}
// Populates the
func populateStoredCdrField(cdr *engine.StoredCdr, fieldId, fieldVal, timezone string) error {
var err error
switch fieldId {
case utils.TOR:
cdr.TOR += fieldVal
case utils.ACCID:
cdr.AccId += fieldVal
case utils.REQTYPE:
cdr.ReqType += fieldVal
case utils.DIRECTION:
cdr.Direction += fieldVal
case utils.TENANT:
cdr.Tenant += fieldVal
case utils.CATEGORY:
cdr.Category += fieldVal
case utils.ACCOUNT:
cdr.Account += fieldVal
case utils.SUBJECT:
cdr.Subject += fieldVal
case utils.DESTINATION:
cdr.Destination += fieldVal
case utils.RATED_FLD:
cdr.Rated, _ = strconv.ParseBool(fieldVal)
case utils.SETUP_TIME:
if cdr.SetupTime, err = utils.ParseTimeDetectLayout(fieldVal, timezone); err != nil {
return fmt.Errorf("Cannot parse answer time field with value: %s, err: %s", fieldVal, err.Error())
}
case utils.PDD:
if cdr.Pdd, err = utils.ParseDurationWithSecs(fieldVal); err != nil {
return fmt.Errorf("Cannot parse answer time field with value: %s, err: %s", fieldVal, err.Error())
}
case utils.ANSWER_TIME:
if cdr.AnswerTime, err = utils.ParseTimeDetectLayout(fieldVal, timezone); err != nil {
return fmt.Errorf("Cannot parse answer time field with value: %s, err: %s", fieldVal, err.Error())
}
case utils.USAGE:
if cdr.Usage, err = utils.ParseDurationWithSecs(fieldVal); err != nil {
return fmt.Errorf("Cannot parse duration field with value: %s, err: %s", fieldVal, err.Error())
}
case utils.SUPPLIER:
cdr.Supplier += fieldVal
case utils.DISCONNECT_CAUSE:
cdr.DisconnectCause += fieldVal
case utils.COST:
if cdr.Cost, err = strconv.ParseFloat(fieldVal, 64); err != nil {
return fmt.Errorf("Cannot parse cost field with value: %s, err: %s", fieldVal, err.Error())
}
default: // Extra fields will not match predefined so they all show up here
cdr.ExtraFields[fieldId] += fieldVal
}
return nil
}
// Converts a record (header or normal) to StoredCdr
func (self *FwvRecordsProcessor) recordToStoredCdr(record string, cfgKey string) (*engine.StoredCdr, error) {
var err error
var lazyHttpFields []*config.CfgCdrField
var cfgFields []*config.CfgCdrField
var duMultiplyFactor float64
var storedCdr *engine.StoredCdr
if self.headerCdr != nil { // Clone the header CDR so we can use it as base to future processing (inherit fields defined there)
storedCdr = self.headerCdr.Clone()
} else {
storedCdr = &engine.StoredCdr{CdrHost: "0.0.0.0", ExtraFields: make(map[string]string), Cost: -1}
}
if cfgKey == "*header" {
cfgFields = self.dfltCfg.HeaderFields
storedCdr.CdrSource = self.dfltCfg.CdrSourceId
duMultiplyFactor = self.dfltCfg.DataUsageMultiplyFactor
} else {
cfgFields = self.cdrcCfgs[cfgKey].ContentFields
storedCdr.CdrSource = self.cdrcCfgs[cfgKey].CdrSourceId
duMultiplyFactor = self.cdrcCfgs[cfgKey].DataUsageMultiplyFactor
}
for _, cdrFldCfg := range cfgFields {
var fieldVal string
switch cdrFldCfg.Type {
case utils.CDRFIELD:
for _, cfgFieldRSR := range cdrFldCfg.Value {
if cfgFieldRSR.IsStatic() {
fieldVal += cfgFieldRSR.ParseValue("")
} else { // Dynamic value extracted using index
if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx {
return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cdrFldCfg.Tag)
} else {
fieldVal += cfgFieldRSR.ParseValue(fwvValue(record, cfgFieldIdx, cdrFldCfg.Width, cdrFldCfg.Padding))
}
}
}
case utils.HTTP_POST:
lazyHttpFields = append(lazyHttpFields, cdrFldCfg) // Will process later so we can send an estimation of storedCdr to http server
default:
//return nil, fmt.Errorf("Unsupported field type: %s", cdrFldCfg.Type)
continue // Don't do anything for unsupported fields
}
if err := populateStoredCdrField(storedCdr, cdrFldCfg.CdrFieldId, fieldVal, self.timezone); err != nil {
return nil, err
}
}
if storedCdr.CgrId == "" && storedCdr.AccId != "" && cfgKey != "*header" {
storedCdr.CgrId = utils.Sha1(storedCdr.AccId, storedCdr.SetupTime.UTC().String())
}
if storedCdr.TOR == utils.DATA && duMultiplyFactor != 0 {
storedCdr.Usage = time.Duration(float64(storedCdr.Usage.Nanoseconds()) * duMultiplyFactor)
}
for _, httpFieldCfg := range lazyHttpFields { // Lazy process the http fields
var outValByte []byte
var fieldVal, httpAddr string
for _, rsrFld := range httpFieldCfg.Value {
httpAddr += rsrFld.ParseValue("")
}
if outValByte, err = utils.HttpJsonPost(httpAddr, self.httpSkipTlsCheck, storedCdr); err != nil && httpFieldCfg.Mandatory {
return nil, err
} else {
fieldVal = string(outValByte)
if len(fieldVal) == 0 && httpFieldCfg.Mandatory {
return nil, fmt.Errorf("MandatoryIeMissing: Empty result for http_post field: %s", httpFieldCfg.Tag)
}
if err := populateStoredCdrField(storedCdr, httpFieldCfg.CdrFieldId, fieldVal, self.timezone); err != nil {
return nil, err
}
}
}
return storedCdr, nil
}
func (fsev FSEvent) AsStoredCdr(timezone string) *engine.StoredCdr {
storCdr := new(engine.StoredCdr)
storCdr.CgrId = fsev.GetCgrId(timezone)
storCdr.TOR = utils.VOICE
storCdr.AccId = fsev.GetUUID()
storCdr.CdrHost = fsev.GetOriginatorIP(utils.META_DEFAULT)
storCdr.CdrSource = "FS_" + fsev.GetName()
storCdr.ReqType = fsev.GetReqType(utils.META_DEFAULT)
storCdr.Direction = fsev.GetDirection(utils.META_DEFAULT)
storCdr.Tenant = fsev.GetTenant(utils.META_DEFAULT)
storCdr.Category = fsev.GetCategory(utils.META_DEFAULT)
storCdr.Account = fsev.GetAccount(utils.META_DEFAULT)
storCdr.Subject = fsev.GetSubject(utils.META_DEFAULT)
storCdr.Destination = fsev.GetDestination(utils.META_DEFAULT)
storCdr.SetupTime, _ = fsev.GetSetupTime(utils.META_DEFAULT, timezone)
storCdr.AnswerTime, _ = fsev.GetAnswerTime(utils.META_DEFAULT, timezone)
storCdr.Usage, _ = fsev.GetDuration(utils.META_DEFAULT)
storCdr.Pdd, _ = fsev.GetPdd(utils.META_DEFAULT)
storCdr.ExtraFields = fsev.GetExtraFields()
storCdr.Cost = -1
storCdr.Supplier = fsev.GetSupplier(utils.META_DEFAULT)
storCdr.DisconnectCause = fsev.GetDisconnectCause(utils.META_DEFAULT)
return storCdr
}
// Write individual cdr into content buffer, build stats
func (cdre *CdrExporter) processCdr(cdr *engine.StoredCdr) error {
if cdr == nil || len(cdr.CgrId) == 0 { // We do not export empty CDRs
return nil
} else if cdr.ExtraFields == nil { // Avoid assignment in nil map if not initialized
cdr.ExtraFields = make(map[string]string)
}
// Cost multiply
if cdre.dataUsageMultiplyFactor != 0.0 && cdr.TOR == utils.DATA {
cdr.UsageMultiply(cdre.dataUsageMultiplyFactor, cdre.cgrPrecision)
} else if cdre.smsUsageMultiplyFactor != 0 && cdr.TOR == utils.SMS {
cdr.UsageMultiply(cdre.smsUsageMultiplyFactor, cdre.cgrPrecision)
} else if cdre.genericUsageMultiplyFactor != 0 && cdr.TOR == utils.GENERIC {
cdr.UsageMultiply(cdre.genericUsageMultiplyFactor, cdre.cgrPrecision)
}
if cdre.costMultiplyFactor != 0.0 {
cdr.CostMultiply(cdre.costMultiplyFactor, cdre.cgrPrecision)
}
var err error
cdrRow := make([]string, len(cdre.exportTemplate.ContentFields))
for idx, cfgFld := range cdre.exportTemplate.ContentFields {
var outVal string
switch cfgFld.Type {
case utils.FILLER:
outVal = cfgFld.Value.Id()
cfgFld.Padding = "right"
case utils.CONSTANT:
outVal = cfgFld.Value.Id()
case utils.CDRFIELD:
outVal, err = cdre.cdrFieldValue(cdr, cfgFld)
case DATETIME:
outVal, err = cdre.getDateTimeFieldVal(cdr, cfgFld)
case utils.HTTP_POST:
var outValByte []byte
httpAddr := cfgFld.Value.Id()
if len(httpAddr) == 0 {
err = fmt.Errorf("Empty http address for field %s type %s", cfgFld.Tag, cfgFld.Type)
} else if outValByte, err = utils.HttpJsonPost(httpAddr, cdre.httpSkipTlsCheck, cdr); err == nil {
outVal = string(outValByte)
if len(outVal) == 0 && cfgFld.Mandatory {
err = fmt.Errorf("Empty result for http_post field: %s", cfgFld.Tag)
}
}
case utils.COMBIMED:
outVal, err = cdre.getCombimedCdrFieldVal(cdr, cfgFld)
case utils.METATAG:
if cfgFld.Value.Id() == META_MASKDESTINATION {
outVal, err = cdre.metaHandler(cfgFld.Value.Id(), cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION}))
} else {
outVal, err = cdre.metaHandler(cfgFld.Value.Id(), cfgFld.Layout)
}
}
if err != nil {
engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with cgrid: %s and runid: %s, error: %s", cdr.CgrId, cdr.MediationRunId, err.Error()))
return err
}
fmtOut := outVal
if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with cgrid: %s, runid: %s, fieldName: %s, fieldValue: %s, error: %s", cdr.CgrId, cdr.MediationRunId, cfgFld.Tag, outVal, err.Error()))
return err
}
cdrRow[idx] += fmtOut
}
if len(cdrRow) == 0 { // No CDR data, most likely no configuration fields defined
return nil
} else {
cdre.content = append(cdre.content, cdrRow)
}
// Done with writing content, compute stats here
if cdre.firstCdrATime.IsZero() || cdr.AnswerTime.Before(cdre.firstCdrATime) {
cdre.firstCdrATime = cdr.AnswerTime
}
if cdr.AnswerTime.After(cdre.lastCdrATime) {
cdre.lastCdrATime = cdr.AnswerTime
}
cdre.numberOfRecords += 1
if cdr.TOR == utils.VOICE { // Only count duration for non data cdrs
cdre.totalDuration += cdr.Usage
}
if cdr.TOR == utils.SMS { // Count usage for SMS
cdre.totalSmsUsage += cdr.Usage
}
if cdr.TOR == utils.GENERIC { // Count usage for GENERIC
cdre.totalGenericUsage += cdr.Usage
}
if cdr.TOR == utils.DATA { // Count usage for DATA
cdre.totalDataUsage += cdr.Usage
}
if cdr.Cost != -1 {
cdre.totalCost += cdr.Cost
cdre.totalCost = utils.Round(cdre.totalCost, cdre.roundDecimals, utils.ROUNDING_MIDDLE)
}
if cdre.firstExpOrderId > cdr.OrderId || cdre.firstExpOrderId == 0 {
cdre.firstExpOrderId = cdr.OrderId
}
if cdre.lastExpOrderId < cdr.OrderId {
cdre.lastExpOrderId = cdr.OrderId
}
return nil
}
// Extracts the value specified by cfgHdr out of cdr
func (cdre *CdrExporter) cdrFieldValue(cdr *engine.StoredCdr, cfgCdrFld *config.CfgCdrField) (string, error) {
for _, fltrRl := range cfgCdrFld.FieldFilter {
if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass {
return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl)
}
}
layout := cfgCdrFld.Layout
if len(layout) == 0 {
layout = time.RFC3339
}
var retVal string // Concatenate the resulting values
for _, rsrFld := range cfgCdrFld.Value {
var cdrVal string
switch rsrFld.Id {
case COST_DETAILS: // Special case when we need to further extract cost_details out of logDb
if cdr.ExtraFields[COST_DETAILS], err = cdre.getCdrCostDetails(cdr.CgrId, cdr.MediationRunId); err != nil {
return "", err
} else {
cdrVal = cdr.FieldAsString(rsrFld)
}
case utils.COST:
cdrVal = cdr.FormatCost(cdre.costShiftDigits, cdre.roundDecimals)
case utils.USAGE:
cdrVal = cdr.FormatUsage(layout)
case utils.SETUP_TIME:
cdrVal = cdr.SetupTime.Format(layout)
case utils.ANSWER_TIME: // Format time based on layout
cdrVal = cdr.AnswerTime.Format(layout)
case utils.DESTINATION:
cdrVal = cdr.FieldAsString(rsrFld)
if cdre.maskLen != -1 && cdre.maskedDestination(cdrVal) {
cdrVal = MaskDestination(cdrVal, cdre.maskLen)
}
default:
cdrVal = cdr.FieldAsString(rsrFld)
}
retVal += cdrVal
}
return retVal, nil
}