// Takes the record out of csv and turns it into storedCdr which can be processed by CDRS
func (self *CsvRecordsProcessor) recordToStoredCdr(record []string, cdrcId string) (*engine.CDR, error) {
storedCdr := &engine.CDR{OriginHost: "0.0.0.0", Source: self.cdrcCfgs[cdrcId].CdrSourceId, ExtraFields: make(map[string]string), Cost: -1}
var err error
var lazyHttpFields []*config.CfgCdrField
for _, cdrFldCfg := range self.cdrcCfgs[cdrcId].ContentFields {
if utils.IsSliceMember([]string{utils.KAM_FLATSTORE, utils.OSIPS_FLATSTORE}, self.dfltCdrcCfg.CdrFormat) { // Hardcode some values in case of flatstore
switch cdrFldCfg.FieldId {
case utils.ACCID:
cdrFldCfg.Value = utils.ParseRSRFieldsMustCompile("3;1;2", utils.INFIELD_SEP) // in case of flatstore, accounting id is made up out of callid, from_tag and to_tag
case utils.USAGE:
cdrFldCfg.Value = utils.ParseRSRFieldsMustCompile(strconv.Itoa(len(record)-1), utils.INFIELD_SEP) // in case of flatstore, last element will be the duration computed by us
}
}
var fieldVal string
if cdrFldCfg.Type == utils.META_COMPOSED {
for _, cfgFieldRSR := range cdrFldCfg.Value {
if cfgFieldRSR.IsStatic() {
fieldVal += cfgFieldRSR.ParseValue("")
} else { // Dynamic value extracted using index
if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx {
return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cdrFldCfg.Tag)
} else {
fieldVal += cfgFieldRSR.ParseValue(record[cfgFieldIdx])
}
}
}
} else if cdrFldCfg.Type == utils.META_HTTP_POST {
lazyHttpFields = append(lazyHttpFields, cdrFldCfg) // Will process later so we can send an estimation of storedCdr to http server
} else {
return nil, fmt.Errorf("Unsupported field type: %s", cdrFldCfg.Type)
}
if err := storedCdr.ParseFieldValue(cdrFldCfg.FieldId, fieldVal, self.timezone); err != nil {
return nil, err
}
}
storedCdr.CGRID = utils.Sha1(storedCdr.OriginID, storedCdr.SetupTime.UTC().String())
if storedCdr.ToR == utils.DATA && self.cdrcCfgs[cdrcId].DataUsageMultiplyFactor != 0 {
storedCdr.Usage = time.Duration(float64(storedCdr.Usage.Nanoseconds()) * self.cdrcCfgs[cdrcId].DataUsageMultiplyFactor)
}
for _, httpFieldCfg := range lazyHttpFields { // Lazy process the http fields
var outValByte []byte
var fieldVal, httpAddr string
for _, rsrFld := range httpFieldCfg.Value {
httpAddr += rsrFld.ParseValue("")
}
if outValByte, err = utils.HttpJsonPost(httpAddr, self.httpSkipTlsCheck, storedCdr); err != nil && httpFieldCfg.Mandatory {
return nil, err
} else {
fieldVal = string(outValByte)
if len(fieldVal) == 0 && httpFieldCfg.Mandatory {
return nil, fmt.Errorf("MandatoryIeMissing: Empty result for http_post field: %s", httpFieldCfg.Tag)
}
if err := storedCdr.ParseFieldValue(httpFieldCfg.FieldId, fieldVal, self.timezone); err != nil {
return nil, err
}
}
}
return storedCdr, nil
}
// Does not block for posts, no error reports
func callUrlAsync(ub *Account, sq *StatsQueueTriggered, a *Action) error {
var o interface{}
if ub != nil {
o = ub
}
if sq != nil {
o = sq
}
jsn, err := json.Marshal(o)
if err != nil {
return err
}
cfg := config.CgrConfig()
go func() {
for i := 0; i < 5; i++ { // Loop so we can increase the success rate on best effort
if _, err = utils.HttpJsonPost(a.ExtraParameters, cfg.HttpSkipTlsVerify, o); err == nil {
break // Success, no need to reinterate
} else if i == 4 { // Last iteration, syslog the warning
Logger.Warning(fmt.Sprintf("<Triggers> WARNING: Failed calling url: [%s], error: [%s], triggered: %s", a.ExtraParameters, err.Error(), jsn))
break
}
time.Sleep(time.Duration(i) * time.Minute)
}
}()
return nil
}
func (cdr *CDR) formatField(cfgFld *config.CfgCdrField, httpSkipTlsCheck bool, groupedCDRs []*CDR) (fmtOut string, err error) {
layout := cfgFld.Layout
if layout == "" {
layout = time.RFC3339
}
var outVal string
switch cfgFld.Type {
case utils.META_FILLER:
outVal = cfgFld.Value.Id()
cfgFld.Padding = "right"
case utils.META_CONSTANT:
outVal = cfgFld.Value.Id()
case utils.MetaDateTime: // Convert the requested field value into datetime with layout
rawVal, err := cdr.exportFieldValue(cfgFld)
if err != nil {
return "", err
}
if dtFld, err := utils.ParseTimeDetectLayout(rawVal, cfgFld.Timezone); err != nil { // Only one rule makes sense here
return "", err
} else {
outVal = dtFld.Format(layout)
}
case utils.META_HTTP_POST:
var outValByte []byte
httpAddr := cfgFld.Value.Id()
jsn, err := json.Marshal(cdr)
if err != nil {
return "", err
}
if len(httpAddr) == 0 {
err = fmt.Errorf("Empty http address for field %s type %s", cfgFld.Tag, cfgFld.Type)
} else if outValByte, err = utils.HttpJsonPost(httpAddr, httpSkipTlsCheck, jsn); err == nil {
outVal = string(outValByte)
if len(outVal) == 0 && cfgFld.Mandatory {
err = fmt.Errorf("Empty result for http_post field: %s", cfgFld.Tag)
}
}
case utils.META_COMBIMED:
outVal, err = cdr.combimedCdrFieldVal(cfgFld, groupedCDRs)
case utils.META_COMPOSED:
outVal, err = cdr.exportFieldValue(cfgFld)
case utils.MetaMaskedDestination:
if len(cfgFld.MaskDestID) != 0 && CachedDestHasPrefix(cfgFld.MaskDestID, cdr.Destination) {
outVal = "1"
} else {
outVal = "0"
}
}
if err != nil {
return "", err
}
return utils.FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory)
}
// Sample HttpJsonPost, more for usage purposes
func TestHttpJsonPost(t *testing.T) {
cdrOut := &ExternalCDR{CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderID: 123, ToR: utils.VOICE, OriginID: "dsafdsaf",
OriginHost: "192.168.1.1",
Source: utils.UNIT_TEST, RequestType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
Category: "call", Account: "account1", Subject: "tgooiscs0014", Destination: "1002",
SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String(), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String(),
RunID: utils.DEFAULT_RUNID,
Usage: "0.00000001", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
}
jsn, _ := json.Marshal(cdrOut)
if _, err := utils.HttpJsonPost("http://localhost:8000", false, jsn); err == nil {
t.Error(err)
}
}
// Sample HttpJsonPost, more for usage purposes
func TestHttpJsonPost(t *testing.T) {
if !*testLocal {
return
}
cdrOut := &ExternalCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
CdrHost: "192.168.1.1",
CdrSource: utils.UNIT_TEST, ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
Category: "call", Account: "account1", Subject: "tgooiscs0014", Destination: "1002",
SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String(), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String(),
MediationRunId: utils.DEFAULT_RUNID,
Usage: "0.00000001", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
}
if _, err := utils.HttpJsonPost("http://localhost:8000", false, cdrOut); err == nil {
t.Error(err)
}
}
func callUrl(ub *Account, sq *StatsQueueTriggered, a *Action) (err error) {
var o interface{}
if ub != nil {
o = ub
}
if sq != nil {
o = sq
}
jsn, err := json.Marshal(o)
if err != nil {
return err
}
cfg := config.CgrConfig()
_, err = utils.HttpJsonPost(a.ExtraParameters, cfg.HttpSkipTlsVerify, jsn)
return err
}
// Converts a record (header or normal) to StoredCdr
func (self *FwvRecordsProcessor) recordToStoredCdr(record string, cfgKey string) (*engine.StoredCdr, error) {
var err error
var lazyHttpFields []*config.CfgCdrField
var cfgFields []*config.CfgCdrField
var duMultiplyFactor float64
var storedCdr *engine.StoredCdr
if self.headerCdr != nil { // Clone the header CDR so we can use it as base to future processing (inherit fields defined there)
storedCdr = self.headerCdr.Clone()
} else {
storedCdr = &engine.StoredCdr{CdrHost: "0.0.0.0", ExtraFields: make(map[string]string), Cost: -1}
}
if cfgKey == "*header" {
cfgFields = self.dfltCfg.HeaderFields
storedCdr.CdrSource = self.dfltCfg.CdrSourceId
duMultiplyFactor = self.dfltCfg.DataUsageMultiplyFactor
} else {
cfgFields = self.cdrcCfgs[cfgKey].ContentFields
storedCdr.CdrSource = self.cdrcCfgs[cfgKey].CdrSourceId
duMultiplyFactor = self.cdrcCfgs[cfgKey].DataUsageMultiplyFactor
}
for _, cdrFldCfg := range cfgFields {
var fieldVal string
switch cdrFldCfg.Type {
case utils.CDRFIELD:
for _, cfgFieldRSR := range cdrFldCfg.Value {
if cfgFieldRSR.IsStatic() {
fieldVal += cfgFieldRSR.ParseValue("")
} else { // Dynamic value extracted using index
if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx {
return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cdrFldCfg.Tag)
} else {
fieldVal += cfgFieldRSR.ParseValue(fwvValue(record, cfgFieldIdx, cdrFldCfg.Width, cdrFldCfg.Padding))
}
}
}
case utils.HTTP_POST:
lazyHttpFields = append(lazyHttpFields, cdrFldCfg) // Will process later so we can send an estimation of storedCdr to http server
default:
//return nil, fmt.Errorf("Unsupported field type: %s", cdrFldCfg.Type)
continue // Don't do anything for unsupported fields
}
if err := populateStoredCdrField(storedCdr, cdrFldCfg.CdrFieldId, fieldVal, self.timezone); err != nil {
return nil, err
}
}
if storedCdr.CgrId == "" && storedCdr.AccId != "" && cfgKey != "*header" {
storedCdr.CgrId = utils.Sha1(storedCdr.AccId, storedCdr.SetupTime.UTC().String())
}
if storedCdr.TOR == utils.DATA && duMultiplyFactor != 0 {
storedCdr.Usage = time.Duration(float64(storedCdr.Usage.Nanoseconds()) * duMultiplyFactor)
}
for _, httpFieldCfg := range lazyHttpFields { // Lazy process the http fields
var outValByte []byte
var fieldVal, httpAddr string
for _, rsrFld := range httpFieldCfg.Value {
httpAddr += rsrFld.ParseValue("")
}
if outValByte, err = utils.HttpJsonPost(httpAddr, self.httpSkipTlsCheck, storedCdr); err != nil && httpFieldCfg.Mandatory {
return nil, err
} else {
fieldVal = string(outValByte)
if len(fieldVal) == 0 && httpFieldCfg.Mandatory {
return nil, fmt.Errorf("MandatoryIeMissing: Empty result for http_post field: %s", httpFieldCfg.Tag)
}
if err := populateStoredCdrField(storedCdr, httpFieldCfg.CdrFieldId, fieldVal, self.timezone); err != nil {
return nil, err
}
}
}
return storedCdr, nil
}
// Write individual cdr into content buffer, build stats
func (cdre *CdrExporter) processCdr(cdr *engine.CDR) error {
if cdr == nil || len(cdr.CGRID) == 0 { // We do not export empty CDRs
return nil
} else if cdr.ExtraFields == nil { // Avoid assignment in nil map if not initialized
cdr.ExtraFields = make(map[string]string)
}
// Cost multiply
if cdre.dataUsageMultiplyFactor != 0.0 && cdr.ToR == utils.DATA {
cdr.UsageMultiply(cdre.dataUsageMultiplyFactor, cdre.cgrPrecision)
} else if cdre.smsUsageMultiplyFactor != 0 && cdr.ToR == utils.SMS {
cdr.UsageMultiply(cdre.smsUsageMultiplyFactor, cdre.cgrPrecision)
} else if cdre.mmsUsageMultiplyFactor != 0 && cdr.ToR == utils.MMS {
cdr.UsageMultiply(cdre.mmsUsageMultiplyFactor, cdre.cgrPrecision)
} else if cdre.genericUsageMultiplyFactor != 0 && cdr.ToR == utils.GENERIC {
cdr.UsageMultiply(cdre.genericUsageMultiplyFactor, cdre.cgrPrecision)
}
if cdre.costMultiplyFactor != 0.0 {
cdr.CostMultiply(cdre.costMultiplyFactor, cdre.cgrPrecision)
}
var err error
cdrRow := make([]string, len(cdre.exportTemplate.ContentFields))
for idx, cfgFld := range cdre.exportTemplate.ContentFields {
var outVal string
switch cfgFld.Type {
case utils.META_FILLER:
outVal = cfgFld.Value.Id()
cfgFld.Padding = "right"
case utils.META_CONSTANT:
outVal = cfgFld.Value.Id()
case utils.META_COMPOSED:
outVal, err = cdre.cdrFieldValue(cdr, cfgFld)
case META_DATETIME:
outVal, err = cdre.getDateTimeFieldVal(cdr, cfgFld)
case utils.META_HTTP_POST:
var outValByte []byte
httpAddr := cfgFld.Value.Id()
if len(httpAddr) == 0 {
err = fmt.Errorf("Empty http address for field %s type %s", cfgFld.Tag, cfgFld.Type)
} else if outValByte, err = utils.HttpJsonPost(httpAddr, cdre.httpSkipTlsCheck, cdr); err == nil {
outVal = string(outValByte)
if len(outVal) == 0 && cfgFld.Mandatory {
err = fmt.Errorf("Empty result for http_post field: %s", cfgFld.Tag)
}
}
case utils.META_COMBIMED:
outVal, err = cdre.getCombimedCdrFieldVal(cdr, cfgFld)
case utils.META_HANDLER:
if cfgFld.Value.Id() == META_MASKDESTINATION {
outVal, err = cdre.metaHandler(cfgFld.Value.Id(), cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION}))
} else {
outVal, err = cdre.metaHandler(cfgFld.Value.Id(), cfgFld.Layout)
}
}
if err != nil {
utils.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with CGRID: %s and runid: %s, error: %s", cdr.CGRID, cdr.RunID, err.Error()))
return err
}
fmtOut := outVal
if fmtOut, err = utils.FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
utils.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with CGRID: %s, runid: %s, fieldName: %s, fieldValue: %s, error: %s", cdr.CGRID, cdr.RunID, cfgFld.Tag, outVal, err.Error()))
return err
}
cdrRow[idx] += fmtOut
}
if len(cdrRow) == 0 { // No CDR data, most likely no configuration fields defined
return nil
} else {
cdre.content = append(cdre.content, cdrRow)
}
// Done with writing content, compute stats here
if cdre.firstCdrATime.IsZero() || cdr.AnswerTime.Before(cdre.firstCdrATime) {
cdre.firstCdrATime = cdr.AnswerTime
}
if cdr.AnswerTime.After(cdre.lastCdrATime) {
cdre.lastCdrATime = cdr.AnswerTime
}
cdre.numberOfRecords += 1
if cdr.ToR == utils.VOICE { // Only count duration for non data cdrs
cdre.totalDuration += cdr.Usage
}
if cdr.ToR == utils.SMS { // Count usage for SMS
cdre.totalSmsUsage += cdr.Usage
}
if cdr.ToR == utils.MMS { // Count usage for MMS
cdre.totalMmsUsage += cdr.Usage
}
if cdr.ToR == utils.GENERIC { // Count usage for GENERIC
cdre.totalGenericUsage += cdr.Usage
}
if cdr.ToR == utils.DATA { // Count usage for DATA
cdre.totalDataUsage += cdr.Usage
}
if cdr.Cost != -1 {
cdre.totalCost += cdr.Cost
cdre.totalCost = utils.Round(cdre.totalCost, cdre.roundDecimals, utils.ROUNDING_MIDDLE)
}
if cdre.firstExpOrderId > cdr.OrderID || cdre.firstExpOrderId == 0 {
cdre.firstExpOrderId = cdr.OrderID
}
if cdre.lastExpOrderId < cdr.OrderID {
cdre.lastExpOrderId = cdr.OrderID
}
return nil
}
// Write individual cdr into content buffer, build stats
func (cdre *CdrExporter) processCdr(cdr *utils.StoredCdr) error {
if cdr == nil || len(cdr.CgrId) == 0 { // We do not export empty CDRs
return nil
}
if cdre.dataUsageMultiplyFactor != 0.0 && cdr.TOR == utils.DATA {
cdr.UsageMultiply(cdre.dataUsageMultiplyFactor, cdre.cgrPrecision)
}
if cdre.costMultiplyFactor != 0.0 {
cdr.CostMultiply(cdre.costMultiplyFactor, cdre.cgrPrecision)
}
var err error
cdrRow := make([]string, len(cdre.exportTemplate.ContentFields))
for idx, cfgFld := range cdre.exportTemplate.ContentFields {
var outVal string
switch cfgFld.Type {
case FILLER:
outVal = cfgFld.Value
cfgFld.Padding = "right"
case CONSTANT:
outVal = cfgFld.Value
case utils.CDRFIELD:
outVal, err = cdre.cdrFieldValue(cdr, cfgFld.Filter, cfgFld.ValueAsRSRField(), cfgFld.Layout)
case DATETIME:
outVal, err = cdre.getDateTimeFieldVal(cdr, cfgFld.Filter, cfgFld.ValueAsRSRField(), cfgFld.Layout)
case HTTP_POST:
var outValByte []byte
if outValByte, err = utils.HttpJsonPost(cfgFld.Value, cdre.httpSkipTlsCheck, cdr); err == nil {
outVal = string(outValByte)
if len(outVal) == 0 && cfgFld.Mandatory {
err = fmt.Errorf("Empty result for http_post field: %s", cfgFld.Name)
}
}
case COMBIMED:
outVal, err = cdre.getCombimedCdrFieldVal(cdr, cfgFld.Filter, cfgFld.ValueAsRSRField())
case CONCATENATED_CDRFIELD:
for _, fld := range strings.Split(cfgFld.Value, ",") {
if fldOut, err := cdre.cdrFieldValue(cdr, cfgFld.Filter, &utils.RSRField{Id: fld}, cfgFld.Layout); err != nil {
break // The error will be reported bellow
} else {
outVal += fldOut
}
}
case METATAG:
if cfgFld.Value == META_MASKDESTINATION {
outVal, err = cdre.metaHandler(cfgFld.Value, cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION}))
} else {
outVal, err = cdre.metaHandler(cfgFld.Value, cfgFld.Layout)
}
}
if err != nil {
engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with cgrid: %s and runid: %s, error: %s", cdr.CgrId, cdr.MediationRunId, err.Error()))
return err
}
fmtOut := outVal
if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with cgrid: %s, runid: %s, fieldName: %s, fieldValue: %s, error: %s", cdr.CgrId, cdr.MediationRunId, cfgFld.Name, outVal, err.Error()))
return err
}
cdrRow[idx] += fmtOut
}
if len(cdrRow) == 0 { // No CDR data, most likely no configuration fields defined
return nil
} else {
cdre.content = append(cdre.content, cdrRow)
}
// Done with writing content, compute stats here
if cdre.firstCdrATime.IsZero() || cdr.AnswerTime.Before(cdre.firstCdrATime) {
cdre.firstCdrATime = cdr.AnswerTime
}
if cdr.AnswerTime.After(cdre.lastCdrATime) {
cdre.lastCdrATime = cdr.AnswerTime
}
cdre.numberOfRecords += 1
if cdr.TOR == utils.VOICE { // Only count duration for non data cdrs
cdre.totalDuration += cdr.Usage
}
if cdr.TOR == utils.SMS { // Count usage for SMS
cdre.totalSmsUsage += cdr.Usage
}
if cdr.TOR == utils.DATA { // Count usage for SMS
cdre.totalDataUsage += cdr.Usage
}
if cdr.Cost != -1 {
cdre.totalCost += cdr.Cost
cdre.totalCost = utils.Round(cdre.totalCost, cdre.roundDecimals, utils.ROUNDING_MIDDLE)
}
if cdre.firstExpOrderId > cdr.OrderId || cdre.firstExpOrderId == 0 {
cdre.firstExpOrderId = cdr.OrderId
}
if cdre.lastExpOrderId < cdr.OrderId {
cdre.lastExpOrderId = cdr.OrderId
}
return nil
}
func (xmlProc *XMLRecordsProcessor) recordToCDR(xmlEntity tree.Res, cdrcCfg *config.CdrcConfig) (*engine.CDR, error) {
cdr := &engine.CDR{OriginHost: "0.0.0.0", Source: cdrcCfg.CdrSourceId, ExtraFields: make(map[string]string), Cost: -1}
var lazyHttpFields []*config.CfgCdrField
var err error
for _, cdrFldCfg := range cdrcCfg.ContentFields {
var fieldVal string
if cdrFldCfg.Type == utils.META_COMPOSED {
for _, cfgFieldRSR := range cdrFldCfg.Value {
if cfgFieldRSR.IsStatic() {
fieldVal += cfgFieldRSR.ParseValue("")
} else { // Dynamic value extracted using path
absolutePath := utils.ParseHierarchyPath(cfgFieldRSR.Id, "")
relPath := utils.HierarchyPath(absolutePath[len(xmlProc.cdrPath)-1:]) // Need relative path to the xmlElmnt
if elmntText, err := elementText(xmlEntity, relPath.AsString("/", true)); err != nil {
return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s, err: %s", xmlEntity, cdrFldCfg.Tag, err.Error())
} else {
fieldVal += cfgFieldRSR.ParseValue(elmntText)
}
}
}
} else if cdrFldCfg.Type == utils.META_HTTP_POST {
lazyHttpFields = append(lazyHttpFields, cdrFldCfg) // Will process later so we can send an estimation of cdr to http server
} else if cdrFldCfg.Type == utils.META_HANDLER && cdrFldCfg.HandlerId == utils.HandlerSubstractUsage {
usage, err := handlerSubstractUsage(xmlEntity, cdrFldCfg.Value, xmlProc.cdrPath, xmlProc.timezone)
if err != nil {
return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s, err: %s", xmlEntity, cdrFldCfg.Tag, err.Error())
}
fieldVal += strconv.FormatFloat(usage.Seconds(), 'f', -1, 64)
} else {
return nil, fmt.Errorf("Unsupported field type: %s", cdrFldCfg.Type)
}
if err := cdr.ParseFieldValue(cdrFldCfg.FieldId, fieldVal, xmlProc.timezone); err != nil {
return nil, err
}
}
cdr.CGRID = utils.Sha1(cdr.OriginID, cdr.SetupTime.UTC().String())
if cdr.ToR == utils.DATA && cdrcCfg.DataUsageMultiplyFactor != 0 {
cdr.Usage = time.Duration(float64(cdr.Usage.Nanoseconds()) * cdrcCfg.DataUsageMultiplyFactor)
}
for _, httpFieldCfg := range lazyHttpFields { // Lazy process the http fields
var outValByte []byte
var fieldVal, httpAddr string
for _, rsrFld := range httpFieldCfg.Value {
httpAddr += rsrFld.ParseValue("")
}
var jsn []byte
jsn, err = json.Marshal(cdr)
if err != nil {
return nil, err
}
if outValByte, err = utils.HttpJsonPost(httpAddr, xmlProc.httpSkipTlsCheck, jsn); err != nil && httpFieldCfg.Mandatory {
return nil, err
} else {
fieldVal = string(outValByte)
if len(fieldVal) == 0 && httpFieldCfg.Mandatory {
return nil, fmt.Errorf("MandatoryIeMissing: Empty result for http_post field: %s", httpFieldCfg.Tag)
}
if err := cdr.ParseFieldValue(httpFieldCfg.FieldId, fieldVal, xmlProc.timezone); err != nil {
return nil, err
}
}
}
return cdr, nil
}
// Used in place where we need to export the CDR based on an export template
// ExportRecord is a []string to keep it compatible with encoding/csv Writer
func (cdr *CDR) AsExportRecord(exportFields []*config.CfgCdrField, costShiftDigits, roundDecimals int, timezone string, httpSkipTlsCheck bool, maskLen int, maskDestID string, groupedCDRs []*CDR) ([]string, error) {
var err error
expRecord := make([]string, len(exportFields))
for idx, cfgFld := range exportFields {
layout := cfgFld.Layout
if len(layout) == 0 {
layout = time.RFC3339
}
var outVal string
switch cfgFld.Type {
case utils.META_FILLER:
outVal = cfgFld.Value.Id()
cfgFld.Padding = "right"
case utils.META_CONSTANT:
outVal = cfgFld.Value.Id()
case utils.MetaDateTime: // Convert the requested field value into datetime with layout
rawVal, err := cdr.exportFieldValue(cfgFld, costShiftDigits, roundDecimals, layout, maskLen, maskDestID)
if err != nil {
return nil, err
}
if dtFld, err := utils.ParseTimeDetectLayout(rawVal, timezone); err != nil { // Only one rule makes sense here
return nil, err
} else {
outVal = dtFld.Format(layout)
}
case utils.META_HTTP_POST:
var outValByte []byte
httpAddr := cfgFld.Value.Id()
jsn, err := json.Marshal(cdr)
if err != nil {
return nil, err
}
if len(httpAddr) == 0 {
err = fmt.Errorf("Empty http address for field %s type %s", cfgFld.Tag, cfgFld.Type)
} else if outValByte, err = utils.HttpJsonPost(httpAddr, httpSkipTlsCheck, jsn); err == nil {
outVal = string(outValByte)
if len(outVal) == 0 && cfgFld.Mandatory {
err = fmt.Errorf("Empty result for http_post field: %s", cfgFld.Tag)
}
}
case utils.META_COMBIMED:
outVal, err = cdr.combimedCdrFieldVal(cfgFld, groupedCDRs)
case utils.META_COMPOSED:
outVal, err = cdr.exportFieldValue(cfgFld, costShiftDigits, roundDecimals, layout, maskLen, maskDestID)
case utils.MetaMaskedDestination:
if len(maskDestID) != 0 && CachedDestHasPrefix(maskDestID, cdr.Destination) {
outVal = "1"
} else {
outVal = "0"
}
}
if err != nil {
return nil, err
}
fmtOut := outVal
if fmtOut, err = utils.FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
return nil, err
}
expRecord[idx] += fmtOut
}
return expRecord, nil
}
// Takes the record out of csv and turns it into storedCdr which can be processed by CDRS
func (self *CsvRecordsProcessor) recordToStoredCdr(record []string, cdrcCfg *config.CdrcConfig) (*engine.CDR, error) {
storedCdr := &engine.CDR{OriginHost: "0.0.0.0", Source: cdrcCfg.CdrSourceId, ExtraFields: make(map[string]string), Cost: -1}
var err error
var lazyHttpFields []*config.CfgCdrField
for _, cdrFldCfg := range cdrcCfg.ContentFields {
filterBreak := false
for _, rsrFilter := range cdrFldCfg.FieldFilter {
if rsrFilter == nil { // Nil filter does not need to match anything
continue
}
if cfgFieldIdx, _ := strconv.Atoi(rsrFilter.Id); len(record) <= cfgFieldIdx {
return nil, fmt.Errorf("Ignoring record: %v - cannot compile field filter %+v", record, rsrFilter)
} else if !rsrFilter.FilterPasses(record[cfgFieldIdx]) {
filterBreak = true
break
}
}
if filterBreak { // Stop processing this field template since it's filters are not matching
continue
}
if utils.IsSliceMember([]string{utils.KAM_FLATSTORE, utils.OSIPS_FLATSTORE}, self.dfltCdrcCfg.CdrFormat) { // Hardcode some values in case of flatstore
switch cdrFldCfg.FieldId {
case utils.ACCID:
cdrFldCfg.Value = utils.ParseRSRFieldsMustCompile("3;1;2", utils.INFIELD_SEP) // in case of flatstore, accounting id is made up out of callid, from_tag and to_tag
case utils.USAGE:
cdrFldCfg.Value = utils.ParseRSRFieldsMustCompile(strconv.Itoa(len(record)-1), utils.INFIELD_SEP) // in case of flatstore, last element will be the duration computed by us
}
}
var fieldVal string
switch cdrFldCfg.Type {
case utils.META_COMPOSED, utils.MetaUnixTimestamp:
for _, cfgFieldRSR := range cdrFldCfg.Value {
if cfgFieldRSR.IsStatic() {
fieldVal += cfgFieldRSR.ParseValue("")
} else { // Dynamic value extracted using index
if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx {
return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cdrFldCfg.Tag)
} else {
strVal := cfgFieldRSR.ParseValue(record[cfgFieldIdx])
if cdrFldCfg.Type == utils.MetaUnixTimestamp {
t, _ := utils.ParseTimeDetectLayout(strVal, self.timezone)
strVal = strconv.Itoa(int(t.Unix()))
}
fieldVal += strVal
}
}
}
case utils.META_HTTP_POST:
lazyHttpFields = append(lazyHttpFields, cdrFldCfg) // Will process later so we can send an estimation of storedCdr to http server
default:
return nil, fmt.Errorf("Unsupported field type: %s", cdrFldCfg.Type)
}
if err := storedCdr.ParseFieldValue(cdrFldCfg.FieldId, fieldVal, self.timezone); err != nil {
return nil, err
}
}
storedCdr.CGRID = utils.Sha1(storedCdr.OriginID, storedCdr.SetupTime.UTC().String())
if storedCdr.ToR == utils.DATA && cdrcCfg.DataUsageMultiplyFactor != 0 {
storedCdr.Usage = time.Duration(float64(storedCdr.Usage.Nanoseconds()) * cdrcCfg.DataUsageMultiplyFactor)
}
for _, httpFieldCfg := range lazyHttpFields { // Lazy process the http fields
var outValByte []byte
var fieldVal, httpAddr string
for _, rsrFld := range httpFieldCfg.Value {
httpAddr += rsrFld.ParseValue("")
}
var jsn []byte
jsn, err = json.Marshal(storedCdr)
if err != nil {
return nil, err
}
if outValByte, err = utils.HttpJsonPost(httpAddr, self.httpSkipTlsCheck, jsn); err != nil && httpFieldCfg.Mandatory {
return nil, err
} else {
fieldVal = string(outValByte)
if len(fieldVal) == 0 && httpFieldCfg.Mandatory {
return nil, fmt.Errorf("MandatoryIeMissing: Empty result for http_post field: %s", httpFieldCfg.Tag)
}
if err := storedCdr.ParseFieldValue(httpFieldCfg.FieldId, fieldVal, self.timezone); err != nil {
return nil, err
}
}
}
return storedCdr, nil
}