func (driver *Sharded) Insert(params database.Params, analytic database.Analytic) error {
// Construct DBPath
dbPath := manager.DBPath{
Name: params.DBName,
Directory: driver.directory,
}
// Push to right shard based on analytic time
shardName := timeToShardName(analytic.Time)
// Construct shard DBPath
shardPath := manager.DBPath{
Name: shardName,
Directory: dbPath.String(),
}
// Get DB from manager
db, err := driver.DBManager.Acquire(shardPath)
if err != nil {
driver.DBManager.Logger.Error("Error executing Insert/Acquire on DB %s: %v\n", shardPath, err)
return &errors.InternalError
}
defer driver.DBManager.Release(db)
// Insert data if everything's OK
err = query.Insert(db.DB, analytic)
if err != nil {
driver.DBManager.Logger.Error("Error executing Insert on DB %s: %v\n", shardPath, err)
return &errors.InsertFailed
}
return nil
}
// Return the list of all shards in a DBPath
func listShards(dbPath manager.DBPath) []string {
folders, err := ioutil.ReadDir(dbPath.String())
if err != nil {
return nil
}
shards := make([]string, 0)
for _, folder := range folders {
shards = append(shards, folder.Name())
}
return shards
}
func (driver *Sharded) BulkInsert(analytics map[string][]database.Analytic) error {
var acquireErr, insertErr error
var db *sqlpool.Resource
// Run a bulk insert query for each database
for dbName, _analytics := range analytics {
// Group database analytics by shards
shardedAnalytics := make(map[string][]database.Analytic)
for _, analytic := range _analytics {
shardName := timeToShardName(analytic.Time)
shardedAnalytics[shardName] = append(shardedAnalytics[shardName], analytic)
}
// Run a bulk insert query for each shard
for shardName, shardAnalytics := range shardedAnalytics {
// Construct DBPath
dbPath := manager.DBPath{
Name: dbName,
Directory: driver.directory,
}
// Construct shard DBPath
shardPath := manager.DBPath{
Name: shardName,
Directory: dbPath.String(),
}
// Get DB from manager
db, acquireErr = driver.DBManager.Acquire(shardPath)
if acquireErr != nil {
driver.DBManager.Logger.Error("Error executing Insert/Acquire on DB %s: %v\n", shardPath, acquireErr)
continue
}
defer driver.DBManager.Release(db)
// Insert data if everything's OK
insertErr = query.BulkInsert(db.DB, shardAnalytics)
if insertErr != nil {
driver.DBManager.Logger.Error("Error executing Insert on DB %s: %v\n", shardPath, insertErr)
}
}
}
if insertErr != nil {
return &errors.InsertFailed
}
if acquireErr != nil {
return &errors.InternalError
}
return nil
}
func (driver *Sharded) Query(params database.Params) (*database.Analytics, error) {
// Construct DBPath
dbPath := manager.DBPath{
Name: params.DBName,
Directory: driver.directory,
}
// Check if DB file exists
dbExists, err := driver.DBManager.DBExists(dbPath)
if err != nil {
driver.DBManager.Logger.Error("Error executing Query/DBExists on DB %s: %v\n", dbPath, err)
return nil, &errors.InternalError
}
// DB doesn't exist
if !dbExists {
return nil, &errors.InvalidDatabaseName
}
// At this point, there should be shards to query
// Get list of shards by reading directory
shards := listShards(dbPath)
analytics := database.Analytics{}
cachedRequest := cachedRequest(params.URL)
// Read from each shard
for _, shardName := range shards {
// Don't include shard if not in timerange
shardInt, err := shardNameToInt(shardName)
if err != nil {
return nil, err
}
startInt, endInt := timeRangeToInt(params.TimeRange)
if shardInt < startInt || shardInt > endInt {
continue
}
// Get result if is cached
var shardAnalytics *database.Analytics
cacheURL, err := formatURLForCache(params.URL, shardInt, startInt, endInt, params.TimeRange)
if err != nil {
return nil, err
}
cached, inCache := driver.cache.Get(cacheURL)
if inCache {
err = json.Unmarshal(cached, &shardAnalytics)
if err != nil {
driver.DBManager.Logger.Error("Error unmarshaling from cache: %v\n", err)
return nil, err
}
} else {
// Else query shard
// Construct each shard DBPath
shardPath := manager.DBPath{
Name: shardName,
Directory: dbPath.String(),
}
// Get DB shard from manager
db, err := driver.DBManager.Acquire(shardPath)
if err != nil {
driver.DBManager.Logger.Error("Error executing Query/Acquire on DB %s: %v\n", shardPath, err)
return nil, &errors.InternalError
}
defer driver.DBManager.Release(db)
// Return query result
shardAnalytics, err = query.Query(db.DB, params.TimeRange)
if err != nil {
driver.DBManager.Logger.Error("Error executing Query on DB %s: %v\n", shardPath, err)
return nil, &errors.InternalError
}
// Set shard result in cache if asked
if cachedRequest {
if data, err := json.Marshal(shardAnalytics); err == nil {
err = driver.cache.Set(cacheURL, data)
if err != nil {
driver.DBManager.Logger.Error("Error adding to cache: %v\n", err)
}
}
}
}
// Add shard result to analytics
for _, analytic := range shardAnalytics.List {
analytics.List = append(analytics.List, analytic)
}
}
return &analytics, nil
}
func (driver *Sharded) GroupBy(params database.Params) (*database.Aggregates, error) {
// Construct DBPath
dbPath := manager.DBPath{
Name: params.DBName,
Directory: driver.directory,
}
// Check if DB file exists
dbExists, err := driver.DBManager.DBExists(dbPath)
if err != nil {
driver.DBManager.Logger.Error("Error executing GroupBy/DBExists on DB %s: %v\n", dbPath, err)
return nil, &errors.InternalError
}
// DB doesn't exist
if !dbExists {
return nil, &errors.InvalidDatabaseName
}
// At this point, there should be shards to query
// Get list of shards by reading directory
shards := listShards(dbPath)
// Aggregated query result
analytics := database.Aggregates{}
// Helper map to aggregate
analyticsMap := map[string]database.Aggregate{}
cachedRequest := cachedRequest(params.URL)
// Read from each shard
for _, shardName := range shards {
// Don't include shard if not in timerange
shardInt, err := shardNameToInt(shardName)
if err != nil {
return nil, err
}
startInt, endInt := timeRangeToInt(params.TimeRange)
if shardInt < startInt || shardInt > endInt {
continue
}
// Get result if is cached
var shardAnalytics *database.Aggregates
cacheURL, err := formatURLForCache(params.URL, shardInt, startInt, endInt, params.TimeRange)
if err != nil {
return nil, err
}
cached, inCache := driver.cache.Get(cacheURL)
if inCache {
err = json.Unmarshal(cached, &shardAnalytics)
if err != nil {
driver.DBManager.Logger.Error("Error unmarshaling from cache: %v\n", err)
return nil, err
}
} else {
// Else query shard
// Construct each shard DBPath
shardPath := manager.DBPath{
Name: shardName,
Directory: dbPath.String(),
}
// Get DB shard from manager
db, err := driver.DBManager.Acquire(shardPath)
if err != nil {
driver.DBManager.Logger.Error("Error executing GroupBy/Acquire on DB %s: %v\n", shardPath, err)
return nil, &errors.InternalError
}
defer driver.DBManager.Release(db)
// Check for unique query parameter to call function accordingly
if params.Unique {
shardAnalytics, err = query.GroupByUniq(db.DB, params.Property, params.TimeRange)
if err != nil {
driver.DBManager.Logger.Error("Error executing GroupByUniq on DB %s: %v\n", shardPath, err)
return nil, &errors.InternalError
}
} else {
shardAnalytics, err = query.GroupBy(db.DB, params.Property, params.TimeRange)
if err != nil {
driver.DBManager.Logger.Error("Error executing GroupBy on DB %s: %v\n", shardPath, err)
return nil, &errors.InternalError
}
}
// Set shard result in cache if asked
if cachedRequest {
if data, err := json.Marshal(shardAnalytics); err == nil {
err = driver.cache.Set(cacheURL, data)
if err != nil {
driver.DBManager.Logger.Error("Error adding to cache: %v\n", err)
}
}
}
}
// Add shard result to analyticsMap
for _, analytic := range shardAnalytics.List {
if total, ok := analyticsMap[analytic.Id]; ok {
total.Total += analytic.Total
total.Unique += analytic.Unique
analyticsMap[analytic.Id] = total
} else {
analyticsMap[analytic.Id] = analytic
}
}
}
// Convert analyticsMap to an AggregateList struct
aggregateList := database.AggregateList{}
for _, analytic := range analyticsMap {
aggregateList = append(aggregateList, analytic)
}
// Sort and set
sort.Sort(aggregateList)
analytics.List = aggregateList
return &analytics, nil
}
