Example #1
0
// Wrapper for querying a Database struct grouped by a property
func GroupBy(db *sql.DB, property string, timeRange *database.TimeRange) (*database.Aggregates, error) {
	// Query
	queryBuilder := sq.
		Select(property, "COUNT(*)").
		From("visits")

	// Add time constraints if timeRange provided
	if timeRange != nil {
		if !timeRange.Start.Equal(time.Time{}) {
			timeQuery := fmt.Sprintf("time >= %d", timeRange.Start.Unix())
			queryBuilder = queryBuilder.Where(timeQuery)
		}
		if !timeRange.End.Equal(time.Time{}) {
			timeQuery := fmt.Sprintf("time <= %d", timeRange.End.Unix())
			queryBuilder = queryBuilder.Where(timeQuery)
		}
	}

	// Set query Group By condition
	query, _, err := queryBuilder.GroupBy(property).ToSql()
	if err != nil {
		return nil, err
	}

	// Exec query
	rows, err := db.Query(query)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	list := database.Aggregates{}
	for rows.Next() {
		aggregate := database.Aggregate{}
		rows.Scan(&aggregate.Id, &aggregate.Total)

		// For countries, get fullname as Label
		if property == "countryCode" {
			aggregate.Label = geoip.GetCountry(aggregate.Id)
		} else {
			aggregate.Label = aggregate.Id
		}

		list.List = append(list.List, aggregate)
	}

	return &list, nil
}
Example #2
0
func (driver *Sharded) GroupBy(params database.Params) (*database.Aggregates, error) {
	// Construct DBPath
	dbPath := manager.DBPath{
		Name:      params.DBName,
		Directory: driver.directory,
	}

	// Check if DB file exists
	dbExists, err := driver.DBManager.DBExists(dbPath)
	if err != nil {
		driver.DBManager.Logger.Error("Error executing GroupBy/DBExists on DB %s: %v\n", dbPath, err)
		return nil, &errors.InternalError
	}

	// DB doesn't exist
	if !dbExists {
		return nil, &errors.InvalidDatabaseName
	}

	// At this point, there should be shards to query
	// Get list of shards by reading directory
	shards := listShards(dbPath)

	// Aggregated query result
	analytics := database.Aggregates{}
	// Helper map to aggregate
	analyticsMap := map[string]database.Aggregate{}

	cachedRequest := cachedRequest(params.URL)

	// Read from each shard
	for _, shardName := range shards {
		// Don't include shard if not in timerange
		shardInt, err := shardNameToInt(shardName)
		if err != nil {
			return nil, err
		}

		startInt, endInt := timeRangeToInt(params.TimeRange)
		if shardInt < startInt || shardInt > endInt {
			continue
		}

		// Get result if is cached
		var shardAnalytics *database.Aggregates

		cacheURL, err := formatURLForCache(params.URL, shardInt, startInt, endInt, params.TimeRange)
		if err != nil {
			return nil, err
		}

		cached, inCache := driver.cache.Get(cacheURL)
		if inCache {
			err = json.Unmarshal(cached, &shardAnalytics)
			if err != nil {
				driver.DBManager.Logger.Error("Error unmarshaling from cache: %v\n", err)
				return nil, err
			}
		} else {
			// Else query shard
			// Construct each shard DBPath
			shardPath := manager.DBPath{
				Name:      shardName,
				Directory: dbPath.String(),
			}

			// Get DB shard from manager
			db, err := driver.DBManager.Acquire(shardPath)
			if err != nil {
				driver.DBManager.Logger.Error("Error executing GroupBy/Acquire on DB %s: %v\n", shardPath, err)
				return nil, &errors.InternalError
			}
			defer driver.DBManager.Release(db)

			// Check for unique query parameter to call function accordingly
			if params.Unique {
				shardAnalytics, err = query.GroupByUniq(db.DB, params.Property, params.TimeRange)
				if err != nil {
					driver.DBManager.Logger.Error("Error executing GroupByUniq on DB %s: %v\n", shardPath, err)
					return nil, &errors.InternalError
				}
			} else {
				shardAnalytics, err = query.GroupBy(db.DB, params.Property, params.TimeRange)
				if err != nil {
					driver.DBManager.Logger.Error("Error executing GroupBy on DB %s: %v\n", shardPath, err)
					return nil, &errors.InternalError
				}
			}

			// Set shard result in cache if asked
			if cachedRequest {
				if data, err := json.Marshal(shardAnalytics); err == nil {
					err = driver.cache.Set(cacheURL, data)
					if err != nil {
						driver.DBManager.Logger.Error("Error adding to cache: %v\n", err)
					}
				}
			}
		}

		// Add shard result to analyticsMap
		for _, analytic := range shardAnalytics.List {
			if total, ok := analyticsMap[analytic.Id]; ok {
				total.Total += analytic.Total
				total.Unique += analytic.Unique
				analyticsMap[analytic.Id] = total
			} else {
				analyticsMap[analytic.Id] = analytic
			}
		}
	}

	// Convert analyticsMap to an AggregateList struct
	aggregateList := database.AggregateList{}
	for _, analytic := range analyticsMap {
		aggregateList = append(aggregateList, analytic)
	}

	// Sort and set
	sort.Sort(aggregateList)
	analytics.List = aggregateList

	return &analytics, nil
}
Example #3
0
// Wrapper for querying a Database struct grouped by a property
func GroupByUniq(db *sql.DB, property string, timeRange *database.TimeRange) (*database.Aggregates, error) {
	// Subquery for counting unique IPs
	subqueryBuilder := sq.
		Select(property, "COUNT(DISTINCT ip) AS uniqueCount").
		From("visits")

	// Add time constraints if timeRange provided
	if timeRange != nil {
		if !timeRange.Start.Equal(time.Time{}) {
			timeQuery := fmt.Sprintf("time >= %d", timeRange.Start.Unix())
			subqueryBuilder = subqueryBuilder.Where(timeQuery)
		}
		if !timeRange.End.Equal(time.Time{}) {
			timeQuery := fmt.Sprintf("time <= %d", timeRange.End.Unix())
			subqueryBuilder = subqueryBuilder.Where(timeQuery)
		}
	}

	// Format subquery
	subquery, _, err := subqueryBuilder.GroupBy(property).ToSql()
	if err != nil {
		return nil, err
	}

	subquery = fmt.Sprintf("(%s) AS subquery", subquery)

	// Query
	tableProperty := fmt.Sprintf("visits.%s", property)
	subqueryProperty := fmt.Sprintf("subquery.%s", property)
	joinClause := fmt.Sprintf("%s ON %s = %s", subquery, tableProperty, subqueryProperty)

	queryBuilder := sq.
		Select(tableProperty, "COUNT(*) AS total", "uniqueCount").
		From("visits").
		Join(joinClause)

	// Add time constraints if timeRange provided
	if timeRange != nil {
		if !timeRange.Start.Equal(time.Time{}) {
			timeQuery := fmt.Sprintf("time >= %d", timeRange.Start.Unix())
			queryBuilder = queryBuilder.Where(timeQuery)
		}
		if !timeRange.End.Equal(time.Time{}) {
			timeQuery := fmt.Sprintf("time <= %d", timeRange.End.Unix())
			queryBuilder = queryBuilder.Where(timeQuery)
		}
	}

	// Format query
	query, _, err := queryBuilder.GroupBy(tableProperty).ToSql()
	if err != nil {
		return nil, err
	}

	// Exec query
	rows, err := db.Query(query)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	// Format results
	list := database.Aggregates{}
	for rows.Next() {
		aggregate := database.Aggregate{}
		rows.Scan(&aggregate.Id, &aggregate.Total, &aggregate.Unique)

		// For countries, get fullname as Label
		if property == "countryCode" {
			aggregate.Label = geoip.GetCountry(aggregate.Id)
		} else {
			aggregate.Label = aggregate.Id
		}

		list.List = append(list.List, aggregate)
	}

	return &list, nil
}