示例#1
0
func mapPairToCSV(colHeader string, inMap PairList, fh *os.File, geo *geoip.GeoIP) int {

	nb, err := fh.WriteString(colHeader + "\n")
	check(err)
	totalBytes := nb
	i := 0

	for _, v1 := range inMap {

		matches := regexp.MustCompile(`([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)`).FindStringSubmatch(v1.Key)
		if len(matches) >= 1 && geo != nil {
			record := geo.GetRecord(v1.Key)
			if record != nil {
				nb, err = fh.WriteString(v1.Key + csvd + strconv.Itoa(v1.Value) + csvd + record.City + csvd + record.CountryName + "\n")
			}
		} else {
			nb, err = fh.WriteString(v1.Key + csvd + strconv.Itoa(v1.Value) + "\n")
		}
		check(err)
		totalBytes += nb
		i++
	}
	fh.Sync()
	return totalBytes
}
示例#2
0
文件: parser.go 项目: a-jain/encore
func parseQueries(queries <-chan *store.Query, geolocator *geoip.GeoIP) <-chan *store.ParsedQuery {
	parsedQueries := make(chan *store.ParsedQuery)
	go func() {
		for query := range queries {
			request, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(query.RawRequest)))
			if err != nil {
				log.Printf("error parsing result request: %v", err)
			}
			clientIp := request.Header.Get("X-Real-Ip")
			if clientIp == "" {
				clientIp = query.RemoteAddr
			}

			var parameters map[string]string
			if err := json.Unmarshal(query.ParametersJson, &parameters); err != nil {
				log.Printf("error parsing query parameters: %v", err)
			}

			parametersNullable := make(map[string]sql.NullString)
			for k, v := range parameters {
				parametersNullable[k] = sql.NullString{
					String: v,
					Valid:  true,
				}
			}

			host, _, err := net.SplitHostPort(clientIp)
			if err != nil {
				host = clientIp
			}

			country, _ := geolocator.GetCountry(host)

			parsedQueries <- &store.ParsedQuery{
				Query:          query.Id,
				MeasurementId:  parameters["measurementId"],
				Timestamp:      query.Timestamp,
				ClientIp:       net.ParseIP(host),
				ClientLocation: country,
				Substrate:      query.Substrate,
				Parameters:     parametersNullable,
			}
		}
		close(parsedQueries)
	}()
	return parsedQueries
}
示例#3
0
文件: parser.go 项目: a-jain/encore
func parseResults(results <-chan *store.Result, geolocator *geoip.GeoIP) <-chan *store.ParsedResult {
	parsedResults := make(chan *store.ParsedResult)
	go func() {
		for result := range results {
			request, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(result.RawRequest)))
			if err != nil {
				log.Printf("error parsing result request: %v", err)
			}
			measurementId := request.URL.Query().Get("cmh-id")
			outcome := request.URL.Query().Get("cmh-result")
			message := request.URL.Query().Get("cmh-message")
			userAgent := request.Header.Get("User-Agent")
			origin := request.Header.Get("Origin")
			referer := request.Header.Get("Referer")
			clientIp := request.Header.Get("X-Real-Ip")
			if clientIp == "" {
				clientIp = result.RemoteAddr
			}

			host, _, err := net.SplitHostPort(clientIp)
			if err != nil {
				host = clientIp
			}

			country, _ := geolocator.GetCountry(host)

			parsedResults <- &store.ParsedResult{
				Result:         result.Id,
				Timestamp:      result.Timestamp,
				MeasurementId:  measurementId,
				Outcome:        outcome,
				Message:        message,
				Origin:         origin,
				Referer:        referer,
				ClientIp:       net.ParseIP(host),
				ClientLocation: country,
				UserAgent:      userAgent,
			}
		}
		close(parsedResults)
	}()
	return parsedResults

}
示例#4
0
func test4(g geoip.GeoIP, ip string) {
	test(func(s string) string { return g.GetCountry(s) }, ip)
}
示例#5
0
func test4(g *geoip.GeoIP, ip string) {
	test(func(s string) (string, int) { return g.GetCountry(s) }, ip)
}
示例#6
0
文件: snowplow.go 项目: Qlean/silvia
func (event *SnowplowEvent) Transform(request []byte, geo *geoip.GeoIP) error {
	event.Body = request

	normString, err := strconv.Unquote(`"` + string(request) + `"`)
	if err != nil {
		return err
	}

	normString = strings.Replace(normString, "\" ", "", -1)

	snowplowRequest := &SnowplowRequest{}
	err = json.Unmarshal([]byte(normString), snowplowRequest)
	if err != nil {
		return err
	}

	// Binding first level nginx fields
	checkStringForNull(snowplowRequest.IPAddress, &event.UserIP)
	event.CollectorTstamp = time.Now()
	// checkStringForNull(snowplowRequest.Referer, &event.PageReferrer)
	checkStringForNull(snowplowRequest.Useragent, &event.Useragent)

	// Binding snowplow data
	snowplowData := snowplowRequest.RequestBody.Data[0]

	// Bind string values
	checkStringForNull(snowplowData.Event, &event.Event)
	checkStringForNull(snowplowData.PageURL, &event.PageURL)
	checkStringForNull(snowplowData.PageTtile, &event.PageTtile)
	checkStringForNull(snowplowData.PageReferrer, &event.PageReferrer)
	checkStringForNull(snowplowData.VTracker, &event.VTracker)
	checkStringForNull(snowplowData.Aid, &event.Aid)
	checkStringForNull(snowplowData.Platform, &event.Platform)
	checkStringForNull(snowplowData.OsTimezone, &event.OsTimezone)
	checkStringForNull(snowplowData.BrLang, &event.BrLang)
	checkStringForNull(snowplowData.DocCharset, &event.DocCharset)
	checkStringForNull(snowplowData.EventID, &event.EventID)
	checkStringForNull(snowplowData.DomainUserID, &event.DomainUserID)
	checkStringForNull(snowplowData.UserFingerprint, &event.UserFingerprint)
	checkStringForNull(snowplowData.SeCategory, &event.SeCategory)
	checkStringForNull(snowplowData.SeAction, &event.SeAction)
	checkStringForNull(snowplowData.SeLabel, &event.SeLabel)
	checkStringForNull(snowplowData.SeProperty, &event.SeProperty)
	checkFloatForNull(snowplowData.SeValue, &event.SeValue)

	// Bind integer values
	checkIntForNull(snowplowData.BrColorDepth, &event.BrColorDepth)
	checkIntForNull(snowplowData.VisitNum, &event.VisitNum)
	checkIntForNull(snowplowData.UserID, &event.UserID)

	// Bind boolean values
	event.BrPDF, _ = strconv.ParseBool(snowplowData.BrPDF)
	event.BrQT, _ = strconv.ParseBool(snowplowData.BrQT)
	event.BrRealPlayer, _ = strconv.ParseBool(snowplowData.BrRealPlayer)
	event.BrWMA, _ = strconv.ParseBool(snowplowData.BrWMA)
	event.BrDir, _ = strconv.ParseBool(snowplowData.BrDir)
	event.BrFlash, _ = strconv.ParseBool(snowplowData.BrFlash)
	event.BrJava, _ = strconv.ParseBool(snowplowData.BrJava)
	event.BrGears, _ = strconv.ParseBool(snowplowData.BrGears)
	event.BrAg, _ = strconv.ParseBool(snowplowData.BrAg)
	event.BrCookies, _ = strconv.ParseBool(snowplowData.BrCookies)

	// Bind screen resolutions
	event.BrViewWidth = snowplowData.BrView.Width
	event.BrViewHeight = snowplowData.BrView.Height

	event.DvceScreenWidth = snowplowData.DvceScreen.Width
	event.DvceScreenHeight = snowplowData.DvceScreen.Height

	event.DocWidth = snowplowData.Doc.Width
	event.DocHeight = snowplowData.Doc.Width

	// Bind time values
	event.DvceTstamp = snowplowData.DvceTstamp.Time

	// Bind Geo values
	geoIPRecord := geo.GetRecord(snowplowRequest.IPAddress)
	if geoIPRecord != nil {
		checkStringForNull(geoIPRecord.CountryName, &event.GeoCountry)
		checkStringForNull(geoIPRecord.Region, &event.GeoRegion)
		checkStringForNull(geoIPRecord.City, &event.GeoCity)
		checkStringForNull(geoIPRecord.PostalCode, &event.GeoZipcode)
		event.GeoLatitude = geoIPRecord.Latitude
		event.GeoLongtitude = geoIPRecord.Longitude
		checkStringForNull(geoIPRecord.Region, &event.GeoRegionName)
	}

	// Bind useragent values

	ua := user_agent.New(snowplowRequest.Useragent)
	event.DvceIsMobile = ua.Mobile()

	checkStringForNull(ua.OS(), &event.OsName)
	checkStringForNull(ua.Platform(), &event.DvceType)

	name, version := ua.Engine()
	checkStringForNull(name+version, &event.BrRndrNgn)

	name, version = ua.Browser()
	checkStringForNull(name, &event.BrName)
	checkStringForNull(version, &event.BrVersion)

	// Bind contexts
	if len(snowplowData.Contexts) > 0 {
		contextsField := &SnowplowContextsJsonField{}
		err := json.Unmarshal([]byte(snowplowData.Contexts), &contextsField)
		if err != nil {
			return err
		} else {
			rawContexts := contextsField.Data[0]
			rawContexts = rawContexts[8 : len(rawContexts)-1]

			// is json?
			var js interface{}
			err := json.Unmarshal([]byte(rawContexts), &js)
			if err != nil {
				rawContexts = contextsField.Data[0]
			}

			checkStringForNull(string(rawContexts), &event.Contexts)
		}

		contexts := &SnowplowContexts{}
		err = json.Unmarshal([]byte(snowplowData.Contexts), &contexts)
		if err != nil {
			return err
		} else {
			contextsData := contexts.Data[0].Data
			checkStringForNull(contextsData.UtmCampaign, &event.UtmCampaign)
			checkStringForNull(contextsData.UtmContent, &event.UtmContent)
			checkStringForNull(contextsData.UtmTerm, &event.UtmTerm)
			checkStringForNull(contextsData.UtmSource, &event.UtmSource)
			checkStringForNull(contextsData.UtmMedium, &event.UtmMedium)
		}
	} else {
		event.Contexts.Valid = false
	}

	// Bind unstructured event
	if len(snowplowData.UnstructEvent) > 0 {
		checkStringForNull(snowplowData.UnstructEvent, &event.UnstructEvent)
	}

	return nil
}