func mapPairToCSV(colHeader string, inMap PairList, fh *os.File, geo *geoip.GeoIP) int {
nb, err := fh.WriteString(colHeader + "\n")
check(err)
totalBytes := nb
i := 0
for _, v1 := range inMap {
matches := regexp.MustCompile(`([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)`).FindStringSubmatch(v1.Key)
if len(matches) >= 1 && geo != nil {
record := geo.GetRecord(v1.Key)
if record != nil {
nb, err = fh.WriteString(v1.Key + csvd + strconv.Itoa(v1.Value) + csvd + record.City + csvd + record.CountryName + "\n")
}
} else {
nb, err = fh.WriteString(v1.Key + csvd + strconv.Itoa(v1.Value) + "\n")
}
check(err)
totalBytes += nb
i++
}
fh.Sync()
return totalBytes
}
func parseQueries(queries <-chan *store.Query, geolocator *geoip.GeoIP) <-chan *store.ParsedQuery {
parsedQueries := make(chan *store.ParsedQuery)
go func() {
for query := range queries {
request, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(query.RawRequest)))
if err != nil {
log.Printf("error parsing result request: %v", err)
}
clientIp := request.Header.Get("X-Real-Ip")
if clientIp == "" {
clientIp = query.RemoteAddr
}
var parameters map[string]string
if err := json.Unmarshal(query.ParametersJson, ¶meters); err != nil {
log.Printf("error parsing query parameters: %v", err)
}
parametersNullable := make(map[string]sql.NullString)
for k, v := range parameters {
parametersNullable[k] = sql.NullString{
String: v,
Valid: true,
}
}
host, _, err := net.SplitHostPort(clientIp)
if err != nil {
host = clientIp
}
country, _ := geolocator.GetCountry(host)
parsedQueries <- &store.ParsedQuery{
Query: query.Id,
MeasurementId: parameters["measurementId"],
Timestamp: query.Timestamp,
ClientIp: net.ParseIP(host),
ClientLocation: country,
Substrate: query.Substrate,
Parameters: parametersNullable,
}
}
close(parsedQueries)
}()
return parsedQueries
}
func parseResults(results <-chan *store.Result, geolocator *geoip.GeoIP) <-chan *store.ParsedResult {
parsedResults := make(chan *store.ParsedResult)
go func() {
for result := range results {
request, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(result.RawRequest)))
if err != nil {
log.Printf("error parsing result request: %v", err)
}
measurementId := request.URL.Query().Get("cmh-id")
outcome := request.URL.Query().Get("cmh-result")
message := request.URL.Query().Get("cmh-message")
userAgent := request.Header.Get("User-Agent")
origin := request.Header.Get("Origin")
referer := request.Header.Get("Referer")
clientIp := request.Header.Get("X-Real-Ip")
if clientIp == "" {
clientIp = result.RemoteAddr
}
host, _, err := net.SplitHostPort(clientIp)
if err != nil {
host = clientIp
}
country, _ := geolocator.GetCountry(host)
parsedResults <- &store.ParsedResult{
Result: result.Id,
Timestamp: result.Timestamp,
MeasurementId: measurementId,
Outcome: outcome,
Message: message,
Origin: origin,
Referer: referer,
ClientIp: net.ParseIP(host),
ClientLocation: country,
UserAgent: userAgent,
}
}
close(parsedResults)
}()
return parsedResults
}
func test4(g geoip.GeoIP, ip string) {
test(func(s string) string { return g.GetCountry(s) }, ip)
}
func test4(g *geoip.GeoIP, ip string) {
test(func(s string) (string, int) { return g.GetCountry(s) }, ip)
}
func (event *SnowplowEvent) Transform(request []byte, geo *geoip.GeoIP) error {
event.Body = request
normString, err := strconv.Unquote(`"` + string(request) + `"`)
if err != nil {
return err
}
normString = strings.Replace(normString, "\" ", "", -1)
snowplowRequest := &SnowplowRequest{}
err = json.Unmarshal([]byte(normString), snowplowRequest)
if err != nil {
return err
}
// Binding first level nginx fields
checkStringForNull(snowplowRequest.IPAddress, &event.UserIP)
event.CollectorTstamp = time.Now()
// checkStringForNull(snowplowRequest.Referer, &event.PageReferrer)
checkStringForNull(snowplowRequest.Useragent, &event.Useragent)
// Binding snowplow data
snowplowData := snowplowRequest.RequestBody.Data[0]
// Bind string values
checkStringForNull(snowplowData.Event, &event.Event)
checkStringForNull(snowplowData.PageURL, &event.PageURL)
checkStringForNull(snowplowData.PageTtile, &event.PageTtile)
checkStringForNull(snowplowData.PageReferrer, &event.PageReferrer)
checkStringForNull(snowplowData.VTracker, &event.VTracker)
checkStringForNull(snowplowData.Aid, &event.Aid)
checkStringForNull(snowplowData.Platform, &event.Platform)
checkStringForNull(snowplowData.OsTimezone, &event.OsTimezone)
checkStringForNull(snowplowData.BrLang, &event.BrLang)
checkStringForNull(snowplowData.DocCharset, &event.DocCharset)
checkStringForNull(snowplowData.EventID, &event.EventID)
checkStringForNull(snowplowData.DomainUserID, &event.DomainUserID)
checkStringForNull(snowplowData.UserFingerprint, &event.UserFingerprint)
checkStringForNull(snowplowData.SeCategory, &event.SeCategory)
checkStringForNull(snowplowData.SeAction, &event.SeAction)
checkStringForNull(snowplowData.SeLabel, &event.SeLabel)
checkStringForNull(snowplowData.SeProperty, &event.SeProperty)
checkFloatForNull(snowplowData.SeValue, &event.SeValue)
// Bind integer values
checkIntForNull(snowplowData.BrColorDepth, &event.BrColorDepth)
checkIntForNull(snowplowData.VisitNum, &event.VisitNum)
checkIntForNull(snowplowData.UserID, &event.UserID)
// Bind boolean values
event.BrPDF, _ = strconv.ParseBool(snowplowData.BrPDF)
event.BrQT, _ = strconv.ParseBool(snowplowData.BrQT)
event.BrRealPlayer, _ = strconv.ParseBool(snowplowData.BrRealPlayer)
event.BrWMA, _ = strconv.ParseBool(snowplowData.BrWMA)
event.BrDir, _ = strconv.ParseBool(snowplowData.BrDir)
event.BrFlash, _ = strconv.ParseBool(snowplowData.BrFlash)
event.BrJava, _ = strconv.ParseBool(snowplowData.BrJava)
event.BrGears, _ = strconv.ParseBool(snowplowData.BrGears)
event.BrAg, _ = strconv.ParseBool(snowplowData.BrAg)
event.BrCookies, _ = strconv.ParseBool(snowplowData.BrCookies)
// Bind screen resolutions
event.BrViewWidth = snowplowData.BrView.Width
event.BrViewHeight = snowplowData.BrView.Height
event.DvceScreenWidth = snowplowData.DvceScreen.Width
event.DvceScreenHeight = snowplowData.DvceScreen.Height
event.DocWidth = snowplowData.Doc.Width
event.DocHeight = snowplowData.Doc.Width
// Bind time values
event.DvceTstamp = snowplowData.DvceTstamp.Time
// Bind Geo values
geoIPRecord := geo.GetRecord(snowplowRequest.IPAddress)
if geoIPRecord != nil {
checkStringForNull(geoIPRecord.CountryName, &event.GeoCountry)
checkStringForNull(geoIPRecord.Region, &event.GeoRegion)
checkStringForNull(geoIPRecord.City, &event.GeoCity)
checkStringForNull(geoIPRecord.PostalCode, &event.GeoZipcode)
event.GeoLatitude = geoIPRecord.Latitude
event.GeoLongtitude = geoIPRecord.Longitude
checkStringForNull(geoIPRecord.Region, &event.GeoRegionName)
}
// Bind useragent values
ua := user_agent.New(snowplowRequest.Useragent)
event.DvceIsMobile = ua.Mobile()
checkStringForNull(ua.OS(), &event.OsName)
checkStringForNull(ua.Platform(), &event.DvceType)
name, version := ua.Engine()
checkStringForNull(name+version, &event.BrRndrNgn)
name, version = ua.Browser()
checkStringForNull(name, &event.BrName)
checkStringForNull(version, &event.BrVersion)
// Bind contexts
if len(snowplowData.Contexts) > 0 {
contextsField := &SnowplowContextsJsonField{}
err := json.Unmarshal([]byte(snowplowData.Contexts), &contextsField)
if err != nil {
return err
} else {
rawContexts := contextsField.Data[0]
rawContexts = rawContexts[8 : len(rawContexts)-1]
// is json?
var js interface{}
err := json.Unmarshal([]byte(rawContexts), &js)
if err != nil {
rawContexts = contextsField.Data[0]
}
checkStringForNull(string(rawContexts), &event.Contexts)
}
contexts := &SnowplowContexts{}
err = json.Unmarshal([]byte(snowplowData.Contexts), &contexts)
if err != nil {
return err
} else {
contextsData := contexts.Data[0].Data
checkStringForNull(contextsData.UtmCampaign, &event.UtmCampaign)
checkStringForNull(contextsData.UtmContent, &event.UtmContent)
checkStringForNull(contextsData.UtmTerm, &event.UtmTerm)
checkStringForNull(contextsData.UtmSource, &event.UtmSource)
checkStringForNull(contextsData.UtmMedium, &event.UtmMedium)
}
} else {
event.Contexts.Valid = false
}
// Bind unstructured event
if len(snowplowData.UnstructEvent) > 0 {
checkStringForNull(snowplowData.UnstructEvent, &event.UnstructEvent)
}
return nil
}