func process(coralName string, data []map[string]interface{}) {
// Transform the data row by row
log.User(uuid, "sponge.process", "# Transforming data to the coral schema.\n")
log.User(uuid, "sponge.process", "# And importing %v documents.", len(data))
// Initialize benchmarking for current table
start := time.Now()
blockStart := time.Now()
blockSize := int64(1000) // number of documents between each report
documents := int64(0)
totalDocuments := int64(len(data))
for _, row := range data {
// output benchmarking for each block of documents
if documents%blockSize == 0 && documents > 0 {
// calculate stats
percentComplete := float64(documents) / float64(totalDocuments) * float64(100)
msSinceStart := time.Since(start).Nanoseconds() / int64(1000000)
msSinceBlock := time.Since(blockStart).Nanoseconds() / int64(1000000)
timeRemaining := int64(float64(time.Since(start).Seconds()) / float64(percentComplete) * float64(100))
//log stats
log.User(uuid, "sponge.process", "%v%% (%v/%v imported) %vms, %vms avg - last %v in %vms, %vms avg -- est time remaining %vs\n", int64(percentComplete), documents, totalDocuments, msSinceStart, msSinceStart/documents, blockSize, msSinceBlock, msSinceBlock/blockSize, int64(timeRemaining))
blockStart = time.Now()
}
documents = documents + 1
// transform the row
id, newRows, err := fiddler.TransformRow(row, coralName)
if err != nil {
log.Error(uuid, "sponge.process", err, "Error when transforming the row %s.", row)
//RECORD to report about failing transformation
if options.ReportOnFailedRecords {
report.Record(coralName, id, "Failing transform data", err)
}
}
// Usually newRows only will have a document but in the case that we have subcollections
// we may get more than one document from a transformation
for _, newRow := range newRows {
log.Dev(uuid, "sponge.process", "Transforming: %v into %v.", row, newRow)
// send the row to pillar
err = coral.AddRow(newRow, coralName)
if err != nil {
log.Error(uuid, "sponge.process", err, "Error when adding a row") // thae row %v to %s.", string(newRow), modelName)
//RECORD to report about failing adding row to coral db
if options.ReportOnFailedRecords {
report.Record(coralName, id, "Failing add row to coral", err)
}
}
}
}
}
func importFromDB(collections []string) {
// var data []map[string]interface{}
for _, name := range collections { // Reads through all the collections whose transformations are in the strategy configuration file
foreignEntity := source.GetForeignEntity(name)
log.User(uuid, "sponge.importAll", "### Reading data to import from %s into collection '%s'. \n", foreignEntity, name)
// Get the data
data, err := dbsource.GetData(name, &options) //options.offset, options.limit, options.orderby, "")
if err != nil {
log.Error(uuid, "sponge.importAll", err, "Get external data for collection %s.", name)
//RECORD to report about failing modelName
if options.ReportOnFailedRecords {
report.Record(name, "", "Failing to get data.", err)
}
continue
}
log.User(uuid, "sponge.importAll", "### Transforming data and sending it to Coral. \n")
//transform and send to pillar the data
process(name, data)
}
}
// Import gets data from report on failed import, transform it and send it to pillar
func importOnlyFailedRecords() { //dbsource source.Sourcer, limit int, offset int, orderby string, thisStrategy string, reportOnFailedRecords bool) {
log.User(uuid, "sponge.importOnlyFailedRecords", "### Reading file of data to import.")
// get the data that needs to be imported
tables, err := report.ReadReport(options.Reportdbfile) //map[model]map[id]interface{}
if err != nil {
log.Error(uuid, "sponge.importOnlyFailedRecords", err, "Getting the rows that will be imported")
}
var data []map[string]interface{}
for table, ids := range tables {
if len(ids) < 1 { // only one ID
// Get the data
log.User(uuid, "sponge.importOnlyFailedRecords", "### Reading data for entity '%s'. \n", table)
data, err = dbsource.GetData(table, &options) //options.offset, options.limit, options.orderby, options.query)
} else {
log.User(uuid, "sponge.importOnlyFailedRecords", "### Reading data for entity '%s', quering '%s'. \n", table, ids)
data, err = dbsource.GetQueryData(table, &options, ids)
}
if err != nil && options.ReportOnFailedRecords {
report.Record(table, ids, "Failing getting data", err)
}
// transform and get data into pillar
process(table, data)
}
}
// ImportType gets ony data related to table, transform it and send it to pillar
func importType(coralEntity string) { //dbsource source.Sourcer, limit int, offset int, orderby string, query string, modelName string, reportOnFailedRecords bool) {
foreignEntity := source.GetForeignEntity(coralEntity)
// Get the data
log.User(uuid, "sponge.importTable", "### Reading data from table '%s'.", foreignEntity)
data, err := dbsource.GetData(foreignEntity, &options) //options.offset, options.limit, options.orderby, options.query)
if err != nil {
log.Error(uuid, "sponge.importAll", err, "Get external data for table %s.", foreignEntity)
//RECORD to report about failing modelName
if options.ReportOnFailedRecords {
report.Record(foreignEntity, "", "Failing to get data", err)
}
return
}
// Transform and send to pillar
process(coralEntity, data)
}