func mapPairToCSV(colHeader string, inMap PairList, fh *os.File, geo *geoip.GeoIP) int {
nb, err := fh.WriteString(colHeader + "\n")
check(err)
totalBytes := nb
i := 0
for _, v1 := range inMap {
matches := regexp.MustCompile(`([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)`).FindStringSubmatch(v1.Key)
if len(matches) >= 1 && geo != nil {
record := geo.GetRecord(v1.Key)
if record != nil {
nb, err = fh.WriteString(v1.Key + csvd + strconv.Itoa(v1.Value) + csvd + record.City + csvd + record.CountryName + "\n")
}
} else {
nb, err = fh.WriteString(v1.Key + csvd + strconv.Itoa(v1.Value) + "\n")
}
check(err)
totalBytes += nb
i++
}
fh.Sync()
return totalBytes
}
func (event *SnowplowEvent) Transform(request []byte, geo *geoip.GeoIP) error {
event.Body = request
normString, err := strconv.Unquote(`"` + string(request) + `"`)
if err != nil {
return err
}
normString = strings.Replace(normString, "\" ", "", -1)
snowplowRequest := &SnowplowRequest{}
err = json.Unmarshal([]byte(normString), snowplowRequest)
if err != nil {
return err
}
// Binding first level nginx fields
checkStringForNull(snowplowRequest.IPAddress, &event.UserIP)
event.CollectorTstamp = time.Now()
// checkStringForNull(snowplowRequest.Referer, &event.PageReferrer)
checkStringForNull(snowplowRequest.Useragent, &event.Useragent)
// Binding snowplow data
snowplowData := snowplowRequest.RequestBody.Data[0]
// Bind string values
checkStringForNull(snowplowData.Event, &event.Event)
checkStringForNull(snowplowData.PageURL, &event.PageURL)
checkStringForNull(snowplowData.PageTtile, &event.PageTtile)
checkStringForNull(snowplowData.PageReferrer, &event.PageReferrer)
checkStringForNull(snowplowData.VTracker, &event.VTracker)
checkStringForNull(snowplowData.Aid, &event.Aid)
checkStringForNull(snowplowData.Platform, &event.Platform)
checkStringForNull(snowplowData.OsTimezone, &event.OsTimezone)
checkStringForNull(snowplowData.BrLang, &event.BrLang)
checkStringForNull(snowplowData.DocCharset, &event.DocCharset)
checkStringForNull(snowplowData.EventID, &event.EventID)
checkStringForNull(snowplowData.DomainUserID, &event.DomainUserID)
checkStringForNull(snowplowData.UserFingerprint, &event.UserFingerprint)
checkStringForNull(snowplowData.SeCategory, &event.SeCategory)
checkStringForNull(snowplowData.SeAction, &event.SeAction)
checkStringForNull(snowplowData.SeLabel, &event.SeLabel)
checkStringForNull(snowplowData.SeProperty, &event.SeProperty)
checkFloatForNull(snowplowData.SeValue, &event.SeValue)
// Bind integer values
checkIntForNull(snowplowData.BrColorDepth, &event.BrColorDepth)
checkIntForNull(snowplowData.VisitNum, &event.VisitNum)
checkIntForNull(snowplowData.UserID, &event.UserID)
// Bind boolean values
event.BrPDF, _ = strconv.ParseBool(snowplowData.BrPDF)
event.BrQT, _ = strconv.ParseBool(snowplowData.BrQT)
event.BrRealPlayer, _ = strconv.ParseBool(snowplowData.BrRealPlayer)
event.BrWMA, _ = strconv.ParseBool(snowplowData.BrWMA)
event.BrDir, _ = strconv.ParseBool(snowplowData.BrDir)
event.BrFlash, _ = strconv.ParseBool(snowplowData.BrFlash)
event.BrJava, _ = strconv.ParseBool(snowplowData.BrJava)
event.BrGears, _ = strconv.ParseBool(snowplowData.BrGears)
event.BrAg, _ = strconv.ParseBool(snowplowData.BrAg)
event.BrCookies, _ = strconv.ParseBool(snowplowData.BrCookies)
// Bind screen resolutions
event.BrViewWidth = snowplowData.BrView.Width
event.BrViewHeight = snowplowData.BrView.Height
event.DvceScreenWidth = snowplowData.DvceScreen.Width
event.DvceScreenHeight = snowplowData.DvceScreen.Height
event.DocWidth = snowplowData.Doc.Width
event.DocHeight = snowplowData.Doc.Width
// Bind time values
event.DvceTstamp = snowplowData.DvceTstamp.Time
// Bind Geo values
geoIPRecord := geo.GetRecord(snowplowRequest.IPAddress)
if geoIPRecord != nil {
checkStringForNull(geoIPRecord.CountryName, &event.GeoCountry)
checkStringForNull(geoIPRecord.Region, &event.GeoRegion)
checkStringForNull(geoIPRecord.City, &event.GeoCity)
checkStringForNull(geoIPRecord.PostalCode, &event.GeoZipcode)
event.GeoLatitude = geoIPRecord.Latitude
event.GeoLongtitude = geoIPRecord.Longitude
checkStringForNull(geoIPRecord.Region, &event.GeoRegionName)
}
// Bind useragent values
ua := user_agent.New(snowplowRequest.Useragent)
event.DvceIsMobile = ua.Mobile()
checkStringForNull(ua.OS(), &event.OsName)
checkStringForNull(ua.Platform(), &event.DvceType)
name, version := ua.Engine()
checkStringForNull(name+version, &event.BrRndrNgn)
name, version = ua.Browser()
checkStringForNull(name, &event.BrName)
checkStringForNull(version, &event.BrVersion)
// Bind contexts
if len(snowplowData.Contexts) > 0 {
contextsField := &SnowplowContextsJsonField{}
err := json.Unmarshal([]byte(snowplowData.Contexts), &contextsField)
if err != nil {
return err
} else {
rawContexts := contextsField.Data[0]
rawContexts = rawContexts[8 : len(rawContexts)-1]
// is json?
var js interface{}
err := json.Unmarshal([]byte(rawContexts), &js)
if err != nil {
rawContexts = contextsField.Data[0]
}
checkStringForNull(string(rawContexts), &event.Contexts)
}
contexts := &SnowplowContexts{}
err = json.Unmarshal([]byte(snowplowData.Contexts), &contexts)
if err != nil {
return err
} else {
contextsData := contexts.Data[0].Data
checkStringForNull(contextsData.UtmCampaign, &event.UtmCampaign)
checkStringForNull(contextsData.UtmContent, &event.UtmContent)
checkStringForNull(contextsData.UtmTerm, &event.UtmTerm)
checkStringForNull(contextsData.UtmSource, &event.UtmSource)
checkStringForNull(contextsData.UtmMedium, &event.UtmMedium)
}
} else {
event.Contexts.Valid = false
}
// Bind unstructured event
if len(snowplowData.UnstructEvent) > 0 {
checkStringForNull(snowplowData.UnstructEvent, &event.UnstructEvent)
}
return nil
}