Example #1
0
// parseAnalytic takes a structures.PostAnalytic from a POST request
// and returns a database.Analytic ready struct to feed the driver
func parseAnalytic(postData PostAnalytic, geolite2 *maxminddb.Reader, log *logger.Logger) database.Analytic {
	// Create Analytic to inject in DB
	analytic := database.Analytic{
		Time:          time.Now(),
		Event:         postData.Event,
		Path:          postData.Path,
		Ip:            postData.Ip,
		Platform:      postData.Platform,
		RefererDomain: postData.RefererDomain,
		CountryCode:   postData.CountryCode,
	}

	var err error
	// Set time from POST data if passed
	if len(postData.Time) > 0 {
		// Try to parse time as an RFC format or a Unix timestamp
		analytic.Time, err = parseTime(postData.Time)
		if err != nil {
			// Reset to current time if format is invalid
			analytic.Time = time.Now()
		}
	}
	analytic.Time = analytic.Time.UTC()

	// Use headers if provided
	if len(postData.Headers) > 0 {
		// Set analytic referer domain
		if analytic.RefererDomain == "" {
			refererHeader := getReferrer(postData.Headers)
			if referrerURL, err := url.ParseRequestURI(refererHeader); err == nil {
				analytic.RefererDomain = referrerURL.Host
			}
		}

		// Extract analytic platform from userAgent
		if analytic.Platform == "" {
			userAgent := getUserAgent(postData.Headers)
			analytic.Platform = utils.Platform(userAgent)
		}
	}

	// Get countryCode from GeoIp
	analytic.CountryCode, err = geoip.GeoIpLookup(geolite2, postData.Ip)
	if err != nil {
		log.Error("Error [%v] looking for countryCode for IP %s", postData.Ip)
	}

	return analytic
}
Example #2
0
// Wrapper for querying a Database struct
func Query(db *sql.DB, timeRange *database.TimeRange) (*database.Analytics, error) {
	// Query
	queryBuilder := sq.
		Select("time", "event", "path", "ip", "platform", "refererDomain", "countryCode").
		From("visits")

	// Add time constraints if timeRange provided
	if timeRange != nil {
		if !timeRange.Start.Equal(time.Time{}) {
			timeQuery := fmt.Sprintf("time >= %d", timeRange.Start.Unix())
			queryBuilder = queryBuilder.Where(timeQuery)
		}
		if !timeRange.End.Equal(time.Time{}) {
			timeQuery := fmt.Sprintf("time <= %d", timeRange.End.Unix())
			queryBuilder = queryBuilder.Where(timeQuery)
		}
	}

	query, _, err := queryBuilder.ToSql()
	if err != nil {
		return nil, err
	}

	// Exec query
	rows, err := db.Query(query)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	analytics := database.Analytics{}
	for rows.Next() {
		analytic := database.Analytic{}
		var analyticTime int64
		rows.Scan(&analyticTime,
			&analytic.Event,
			&analytic.Path,
			&analytic.Ip,
			&analytic.Platform,
			&analytic.RefererDomain,
			&analytic.CountryCode)

		analytic.Time = time.Unix(analyticTime, 0).UTC()
		analytics.List = append(analytics.List, analytic)
	}

	return &analytics, nil
}
Example #3
0
func NewRouter(opts RouterOpts) (http.Handler, error) {
	// Create the app router
	r := mux.NewRouter()
	var log = logger.New("[Router]")

	geolite2 := opts.Geolite2Reader

	// Initiate DB driver
	driver, err := sqlite.NewShardedDriver(opts.DriverOpts)
	if err != nil {
		return nil, err
	}

	/////
	// Query a DB over time
	/////
	r.Path("/{dbName}/time").
		Methods("GET").
		HandlerFunc(func(w http.ResponseWriter, req *http.Request) {

			// Get params from URL
			vars := mux.Vars(req)
			dbName := vars["dbName"]

			// Parse request query
			if err := req.ParseForm(); err != nil {
				renderError(w, err)
				return
			}

			// Get timeRange if provided
			startTime := req.Form.Get("start")
			endTime := req.Form.Get("end")
			intervalStr := req.Form.Get("interval")

			// Convert startTime and endTime to a TimeRange
			timeRange, err := newTimeRange(startTime, endTime)
			if err != nil {
				renderError(w, &webErrors.InvalidTimeFormat)
				return
			}

			// Cast interval to an integer
			// Defaults to 1 day
			interval := 24 * 60 * 60
			if len(intervalStr) > 0 {
				interval, err = strconv.Atoi(intervalStr)
				if err != nil {
					renderError(w, &webErrors.InvalidInterval)
					return
				}
			}

			unique := false
			if strings.Compare(req.Form.Get("unique"), "true") == 0 {
				unique = true
			}

			// Construct Params object
			params := database.Params{
				DBName:    dbName,
				Interval:  interval,
				TimeRange: timeRange,
				Unique:    unique,
				URL:       req.URL,
			}

			analytics, err := driver.Series(params)
			if err != nil {
				renderError(w, normalizeDriverError(err))
				return
			}

			// Return query result
			render(w, analytics, nil)
		})

	/////
	// Count for a DB
	/////
	r.Path("/{dbName}/count").
		Methods("GET").
		HandlerFunc(func(w http.ResponseWriter, req *http.Request) {

			// Get params from URL
			vars := mux.Vars(req)
			dbName := vars["dbName"]

			// Parse request query
			if err := req.ParseForm(); err != nil {
				renderError(w, err)
				return
			}

			// Get timeRange if provided
			startTime := req.Form.Get("start")
			endTime := req.Form.Get("end")

			// Convert startTime and endTime to a TimeRange
			timeRange, err := newTimeRange(startTime, endTime)
			if err != nil {
				renderError(w, &webErrors.InvalidTimeFormat)
				return
			}

			unique := false
			if strings.Compare(req.Form.Get("unique"), "true") == 0 {
				unique = true
			}

			// Construct Params object
			params := database.Params{
				DBName:    dbName,
				TimeRange: timeRange,
				Unique:    unique,
				URL:       req.URL,
			}

			analytics, err := driver.Count(params)
			if err != nil {
				renderError(w, normalizeDriverError(err))
				return
			}

			// Return query result
			render(w, analytics, nil)
		})

	/////
	// Query a DB by property
	/////
	r.Path("/{dbName}/{property}").
		Methods("GET").
		HandlerFunc(func(w http.ResponseWriter, req *http.Request) {

			// Map allowed requests w/ columns names in DB schema
			allowedProperties := map[string]string{
				"countries": "countryCode",
				"platforms": "platform",
				"domains":   "refererDomain",
				"events":    "event",
			}
			// Get params from URL
			vars := mux.Vars(req)
			dbName := vars["dbName"]
			property := vars["property"]

			// Check that property is allowed to be queried
			property, ok := allowedProperties[property]
			if !ok {
				renderError(w, &webErrors.InvalidProperty)
				return
			}

			// Parse request query
			if err := req.ParseForm(); err != nil {
				renderError(w, err)
				return
			}

			// Get timeRange if provided
			startTime := req.Form.Get("start")
			endTime := req.Form.Get("end")

			timeRange, err := newTimeRange(startTime, endTime)
			if err != nil {
				renderError(w, &webErrors.InvalidTimeFormat)
				return
			}

			unique := false
			if strings.Compare(req.Form.Get("unique"), "true") == 0 {
				unique = true
			}

			// Construct Params object
			params := database.Params{
				DBName:    dbName,
				Property:  property,
				TimeRange: timeRange,
				Unique:    unique,
				URL:       req.URL,
			}

			analytics, err := driver.GroupBy(params)
			if err != nil {
				renderError(w, normalizeDriverError(err))
				return
			}

			// Return query result
			render(w, analytics, nil)
		})

	/////
	// Full query a DB
	/////
	r.Path("/{dbName}").
		Methods("GET").
		HandlerFunc(func(w http.ResponseWriter, req *http.Request) {

			// Get dbName from URL
			vars := mux.Vars(req)
			dbName := vars["dbName"]

			// Parse request query
			if err := req.ParseForm(); err != nil {
				renderError(w, err)
				return
			}

			// Get timeRange if provided
			startTime := req.Form.Get("start")
			endTime := req.Form.Get("end")

			timeRange, err := newTimeRange(startTime, endTime)
			if err != nil {
				renderError(w, &webErrors.InvalidTimeFormat)
				return
			}

			// Construct Params object
			params := database.Params{
				DBName:    dbName,
				TimeRange: timeRange,
				URL:       req.URL,
			}

			analytics, err := driver.Query(params)
			if err != nil {
				renderError(w, normalizeDriverError(err))
				return
			}

			render(w, analytics, nil)
		})

	/////
	// Push a list of analytics to different DBs
	/////
	r.Path("/bulk").
		Methods("POST").
		HandlerFunc(func(w http.ResponseWriter, req *http.Request) {

			// Parse JSON POST data
			postList := PostAnalytics{}
			jsonDecoder := json.NewDecoder(req.Body)
			err := jsonDecoder.Decode(&postList)

			// Invalid JSON
			if err != nil {
				log.Error("Invalid JSON format")
				log.Error("%v", err)
				renderError(w, &webErrors.InvalidJSON)
				return
			}

			// Group analytics by website
			analytics := make(map[string][]database.Analytic)

			for _, postData := range postList.List {
				// Skip analytic if website parameter missing
				if postData.Website == "" {
					log.Error("Skipping analytic: website parameter missing on POST data")
					continue
				}

				// Parse data
				analytic := parseAnalytic(postData, geolite2, log)

				// Add to list
				analytics[postData.Website] = append(analytics[postData.Website], analytic)
			}

			// Insert
			err = driver.BulkInsert(analytics)
			if err != nil {
				renderError(w, normalizeDriverError(err))
				return
			}

			log.Info("Successfully inserted analytics: %#v", analytics)

			render(w, nil, nil)
		})

	/////
	// Push analytics to a specific DB
	/////
	r.Path("/{dbName}").
		Methods("POST").
		HandlerFunc(func(w http.ResponseWriter, req *http.Request) {

			// Get dbName from URL
			vars := mux.Vars(req)
			dbName := vars["dbName"]

			// Parse JSON POST data
			postData := PostData{}
			jsonDecoder := json.NewDecoder(req.Body)
			err := jsonDecoder.Decode(&postData)

			// Invalid JSON
			if err != nil {
				log.Error("Invalid JSON format")
				log.Error("%v", err)
				renderError(w, &webErrors.InvalidJSON)
				return
			}

			// Create Analytic to inject in DB
			analytic := database.Analytic{
				Time:  time.Now(),
				Event: postData.Event,
				Path:  postData.Path,
				Ip:    postData.Ip,
			}

			// Set time from POST data if passed
			if len(postData.Time) > 0 {
				analytic.Time, _ = time.Parse(time.RFC3339, postData.Time)
			}
			analytic.Time = analytic.Time.UTC()

			// Set analytic referer domain
			refererHeader := getReferrer(postData.Headers)
			if referrerURL, err := url.ParseRequestURI(refererHeader); err == nil {
				analytic.RefererDomain = referrerURL.Host
			}

			// Extract analytic platform from userAgent
			userAgent := getUserAgent(postData.Headers)
			analytic.Platform = utils.Platform(userAgent)

			// Get countryCode from GeoIp
			analytic.CountryCode, err = geoip.GeoIpLookup(geolite2, postData.Ip)

			// Construct Params object
			params := database.Params{
				DBName: dbName,
			}

			err = driver.Insert(params, analytic)
			if err != nil {
				renderError(w, normalizeDriverError(err))
				return
			}

			log.Info("Successfully inserted analytic: %#v", analytic)

			render(w, nil, nil)
		})

	/////
	// Push a list of analytics to a specific DB
	/////
	r.Path("/{dbName}/bulk").
		Methods("POST").
		HandlerFunc(func(w http.ResponseWriter, req *http.Request) {

			// Get dbName from URL
			vars := mux.Vars(req)
			dbName := vars["dbName"]

			// Parse JSON POST data
			postList := PostAnalytics{}
			jsonDecoder := json.NewDecoder(req.Body)
			err := jsonDecoder.Decode(&postList)

			// Invalid JSON
			if err != nil {
				log.Error("Invalid JSON format:")
				log.Error("%v", err)
				renderError(w, &webErrors.InvalidJSON)
				return
			}

			// Create map of analytics for Bulk insert
			analytics := make(map[string][]database.Analytic, 1)

			for _, postData := range postList.List {
				// Parse data
				analytic := parseAnalytic(postData, geolite2, log)

				// Add analytic to list
				analytics[dbName] = append(analytics[dbName], analytic)
			}

			// Insert
			err = driver.BulkInsert(analytics)
			if err != nil {
				renderError(w, normalizeDriverError(err))
				return
			}

			log.Info("Successfully inserted analytics: %#v", analytics)

			render(w, nil, nil)
		})

	/////
	// Delete a DB
	/////
	r.Path("/{dbName}").
		Methods("DELETE").
		HandlerFunc(func(w http.ResponseWriter, req *http.Request) {

			// Get dbName from URL
			vars := mux.Vars(req)
			dbName := vars["dbName"]

			// Construct Params object
			params := database.Params{
				DBName: dbName,
			}

			err := driver.Delete(params)
			if err != nil {
				renderError(w, normalizeDriverError(err))
				return
			}

			render(w, nil, nil)
		})

	return r, nil
}