Example #1
0
func mapPairToCSV(colHeader string, inMap PairList, fh *os.File, geo *geoip.GeoIP) int {

	nb, err := fh.WriteString(colHeader + "\n")
	check(err)
	totalBytes := nb
	i := 0

	for _, v1 := range inMap {

		matches := regexp.MustCompile(`([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)`).FindStringSubmatch(v1.Key)
		if len(matches) >= 1 && geo != nil {
			record := geo.GetRecord(v1.Key)
			if record != nil {
				nb, err = fh.WriteString(v1.Key + csvd + strconv.Itoa(v1.Value) + csvd + record.City + csvd + record.CountryName + "\n")
			}
		} else {
			nb, err = fh.WriteString(v1.Key + csvd + strconv.Itoa(v1.Value) + "\n")
		}
		check(err)
		totalBytes += nb
		i++
	}
	fh.Sync()
	return totalBytes
}
Example #2
0
func (event *SnowplowEvent) Transform(request []byte, geo *geoip.GeoIP) error {
	event.Body = request

	normString, err := strconv.Unquote(`"` + string(request) + `"`)
	if err != nil {
		return err
	}

	normString = strings.Replace(normString, "\" ", "", -1)

	snowplowRequest := &SnowplowRequest{}
	err = json.Unmarshal([]byte(normString), snowplowRequest)
	if err != nil {
		return err
	}

	// Binding first level nginx fields
	checkStringForNull(snowplowRequest.IPAddress, &event.UserIP)
	event.CollectorTstamp = time.Now()
	// checkStringForNull(snowplowRequest.Referer, &event.PageReferrer)
	checkStringForNull(snowplowRequest.Useragent, &event.Useragent)

	// Binding snowplow data
	snowplowData := snowplowRequest.RequestBody.Data[0]

	// Bind string values
	checkStringForNull(snowplowData.Event, &event.Event)
	checkStringForNull(snowplowData.PageURL, &event.PageURL)
	checkStringForNull(snowplowData.PageTtile, &event.PageTtile)
	checkStringForNull(snowplowData.PageReferrer, &event.PageReferrer)
	checkStringForNull(snowplowData.VTracker, &event.VTracker)
	checkStringForNull(snowplowData.Aid, &event.Aid)
	checkStringForNull(snowplowData.Platform, &event.Platform)
	checkStringForNull(snowplowData.OsTimezone, &event.OsTimezone)
	checkStringForNull(snowplowData.BrLang, &event.BrLang)
	checkStringForNull(snowplowData.DocCharset, &event.DocCharset)
	checkStringForNull(snowplowData.EventID, &event.EventID)
	checkStringForNull(snowplowData.DomainUserID, &event.DomainUserID)
	checkStringForNull(snowplowData.UserFingerprint, &event.UserFingerprint)
	checkStringForNull(snowplowData.SeCategory, &event.SeCategory)
	checkStringForNull(snowplowData.SeAction, &event.SeAction)
	checkStringForNull(snowplowData.SeLabel, &event.SeLabel)
	checkStringForNull(snowplowData.SeProperty, &event.SeProperty)
	checkFloatForNull(snowplowData.SeValue, &event.SeValue)

	// Bind integer values
	checkIntForNull(snowplowData.BrColorDepth, &event.BrColorDepth)
	checkIntForNull(snowplowData.VisitNum, &event.VisitNum)
	checkIntForNull(snowplowData.UserID, &event.UserID)

	// Bind boolean values
	event.BrPDF, _ = strconv.ParseBool(snowplowData.BrPDF)
	event.BrQT, _ = strconv.ParseBool(snowplowData.BrQT)
	event.BrRealPlayer, _ = strconv.ParseBool(snowplowData.BrRealPlayer)
	event.BrWMA, _ = strconv.ParseBool(snowplowData.BrWMA)
	event.BrDir, _ = strconv.ParseBool(snowplowData.BrDir)
	event.BrFlash, _ = strconv.ParseBool(snowplowData.BrFlash)
	event.BrJava, _ = strconv.ParseBool(snowplowData.BrJava)
	event.BrGears, _ = strconv.ParseBool(snowplowData.BrGears)
	event.BrAg, _ = strconv.ParseBool(snowplowData.BrAg)
	event.BrCookies, _ = strconv.ParseBool(snowplowData.BrCookies)

	// Bind screen resolutions
	event.BrViewWidth = snowplowData.BrView.Width
	event.BrViewHeight = snowplowData.BrView.Height

	event.DvceScreenWidth = snowplowData.DvceScreen.Width
	event.DvceScreenHeight = snowplowData.DvceScreen.Height

	event.DocWidth = snowplowData.Doc.Width
	event.DocHeight = snowplowData.Doc.Width

	// Bind time values
	event.DvceTstamp = snowplowData.DvceTstamp.Time

	// Bind Geo values
	geoIPRecord := geo.GetRecord(snowplowRequest.IPAddress)
	if geoIPRecord != nil {
		checkStringForNull(geoIPRecord.CountryName, &event.GeoCountry)
		checkStringForNull(geoIPRecord.Region, &event.GeoRegion)
		checkStringForNull(geoIPRecord.City, &event.GeoCity)
		checkStringForNull(geoIPRecord.PostalCode, &event.GeoZipcode)
		event.GeoLatitude = geoIPRecord.Latitude
		event.GeoLongtitude = geoIPRecord.Longitude
		checkStringForNull(geoIPRecord.Region, &event.GeoRegionName)
	}

	// Bind useragent values

	ua := user_agent.New(snowplowRequest.Useragent)
	event.DvceIsMobile = ua.Mobile()

	checkStringForNull(ua.OS(), &event.OsName)
	checkStringForNull(ua.Platform(), &event.DvceType)

	name, version := ua.Engine()
	checkStringForNull(name+version, &event.BrRndrNgn)

	name, version = ua.Browser()
	checkStringForNull(name, &event.BrName)
	checkStringForNull(version, &event.BrVersion)

	// Bind contexts
	if len(snowplowData.Contexts) > 0 {
		contextsField := &SnowplowContextsJsonField{}
		err := json.Unmarshal([]byte(snowplowData.Contexts), &contextsField)
		if err != nil {
			return err
		} else {
			rawContexts := contextsField.Data[0]
			rawContexts = rawContexts[8 : len(rawContexts)-1]

			// is json?
			var js interface{}
			err := json.Unmarshal([]byte(rawContexts), &js)
			if err != nil {
				rawContexts = contextsField.Data[0]
			}

			checkStringForNull(string(rawContexts), &event.Contexts)
		}

		contexts := &SnowplowContexts{}
		err = json.Unmarshal([]byte(snowplowData.Contexts), &contexts)
		if err != nil {
			return err
		} else {
			contextsData := contexts.Data[0].Data
			checkStringForNull(contextsData.UtmCampaign, &event.UtmCampaign)
			checkStringForNull(contextsData.UtmContent, &event.UtmContent)
			checkStringForNull(contextsData.UtmTerm, &event.UtmTerm)
			checkStringForNull(contextsData.UtmSource, &event.UtmSource)
			checkStringForNull(contextsData.UtmMedium, &event.UtmMedium)
		}
	} else {
		event.Contexts.Valid = false
	}

	// Bind unstructured event
	if len(snowplowData.UnstructEvent) > 0 {
		checkStringForNull(snowplowData.UnstructEvent, &event.UnstructEvent)
	}

	return nil
}