コード例 #1
0
ファイル: sponge.go プロジェクト: coralproject/sponge
func process(coralName string, data []map[string]interface{}) {

	// Transform the data row by row
	log.User(uuid, "sponge.process", "# Transforming data to the coral schema.\n")
	log.User(uuid, "sponge.process", "# And importing %v documents.", len(data))

	// Initialize benchmarking for current table
	start := time.Now()
	blockStart := time.Now()
	blockSize := int64(1000) // number of documents between each report
	documents := int64(0)
	totalDocuments := int64(len(data))

	for _, row := range data {

		// output benchmarking for each block of documents
		if documents%blockSize == 0 && documents > 0 {

			// calculate stats
			percentComplete := float64(documents) / float64(totalDocuments) * float64(100)
			msSinceStart := time.Since(start).Nanoseconds() / int64(1000000)
			msSinceBlock := time.Since(blockStart).Nanoseconds() / int64(1000000)
			timeRemaining := int64(float64(time.Since(start).Seconds()) / float64(percentComplete) * float64(100))

			//log stats
			log.User(uuid, "sponge.process", "%v%% (%v/%v imported) %vms, %vms avg - last %v in %vms, %vms avg -- est time remaining %vs\n", int64(percentComplete), documents, totalDocuments, msSinceStart, msSinceStart/documents, blockSize, msSinceBlock, msSinceBlock/blockSize, int64(timeRemaining))
			blockStart = time.Now()

		}
		documents = documents + 1

		// transform the row
		id, newRows, err := fiddler.TransformRow(row, coralName)
		if err != nil {
			log.Error(uuid, "sponge.process", err, "Error when transforming the row %s.", row)
			//RECORD to report about failing transformation
			if options.ReportOnFailedRecords {
				report.Record(coralName, id, "Failing transform data", err)
			}
		}

		// Usually newRows only will have a document but in the case that we have subcollections
		// we may get more than one document from a transformation
		for _, newRow := range newRows {

			log.Dev(uuid, "sponge.process", "Transforming: %v into %v.", row, newRow)

			// send the row to pillar
			err = coral.AddRow(newRow, coralName)
			if err != nil {
				log.Error(uuid, "sponge.process", err, "Error when adding a row") // thae row %v to %s.", string(newRow), modelName)
				//RECORD to report about failing adding row to coral db
				if options.ReportOnFailedRecords {
					report.Record(coralName, id, "Failing add row to coral", err)
				}
			}
		}
	}
}
コード例 #2
0
ファイル: sponge.go プロジェクト: coralproject/sponge
func importFromDB(collections []string) {
	// var data []map[string]interface{}

	for _, name := range collections { // Reads through all the collections whose transformations are in the strategy configuration file

		foreignEntity := source.GetForeignEntity(name)

		log.User(uuid, "sponge.importAll", "### Reading data to import from %s into collection '%s'. \n", foreignEntity, name)

		// Get the data
		data, err := dbsource.GetData(name, &options) //options.offset, options.limit, options.orderby, "")
		if err != nil {
			log.Error(uuid, "sponge.importAll", err, "Get external data for collection %s.", name)
			//RECORD to report about failing modelName
			if options.ReportOnFailedRecords {
				report.Record(name, "", "Failing to get data.", err)
			}
			continue
		}

		log.User(uuid, "sponge.importAll", "### Transforming data and sending it to Coral. \n")
		//transform and send to pillar the data
		process(name, data)
	}
}
コード例 #3
0
ファイル: sponge.go プロジェクト: coralproject/sponge
// Import gets data from report on failed import, transform it and send it to pillar
func importOnlyFailedRecords() { //dbsource source.Sourcer, limit int, offset int, orderby string, thisStrategy string, reportOnFailedRecords bool) {

	log.User(uuid, "sponge.importOnlyFailedRecords", "### Reading file of data to import.")

	// get the data that needs to be imported
	tables, err := report.ReadReport(options.Reportdbfile) //map[model]map[id]interface{}
	if err != nil {
		log.Error(uuid, "sponge.importOnlyFailedRecords", err, "Getting the rows that will be imported")
	}

	var data []map[string]interface{}

	for table, ids := range tables {

		if len(ids) < 1 { // only one ID
			// Get the data
			log.User(uuid, "sponge.importOnlyFailedRecords", "### Reading data for entity '%s'. \n", table)
			data, err = dbsource.GetData(table, &options) //options.offset, options.limit, options.orderby, options.query)
		} else {
			log.User(uuid, "sponge.importOnlyFailedRecords", "### Reading data for entity '%s', quering '%s'. \n", table, ids)
			data, err = dbsource.GetQueryData(table, &options, ids)
		}
		if err != nil && options.ReportOnFailedRecords {
			report.Record(table, ids, "Failing getting data", err)
		}

		// transform and get data into pillar
		process(table, data)
	}
}
コード例 #4
0
ファイル: sponge.go プロジェクト: coralproject/sponge
// ImportType gets ony data related to table, transform it and send it to pillar
func importType(coralEntity string) { //dbsource source.Sourcer, limit int, offset int, orderby string, query string, modelName string, reportOnFailedRecords bool) {

	foreignEntity := source.GetForeignEntity(coralEntity)
	// Get the data
	log.User(uuid, "sponge.importTable", "### Reading data from table '%s'.", foreignEntity)

	data, err := dbsource.GetData(foreignEntity, &options) //options.offset, options.limit, options.orderby, options.query)
	if err != nil {
		log.Error(uuid, "sponge.importAll", err, "Get external data for table %s.", foreignEntity)
		//RECORD to report about failing modelName
		if options.ReportOnFailedRecords {
			report.Record(foreignEntity, "", "Failing to get data", err)
		}
		return
	}

	// Transform and send to pillar
	process(coralEntity, data)

}