// ProcessData defers to util.SQLInsertData
func (s *SQLWriter) ProcessData(d data.JSON, outputChan chan data.JSON, killChan chan error) {
// handle panics a bit more gracefully
defer func() {
if err := recover(); err != nil {
util.KillPipelineIfErr(err.(error), killChan)
}
}()
// First check for SQLWriterData
var wd SQLWriterData
err := data.ParseJSONSilent(d, &wd)
logger.Info("SQLWriter: Writing data...")
if err == nil && wd.TableName != "" && wd.InsertData != nil {
logger.Debug("SQLWriter: SQLWriterData scenario")
dd, err := data.NewJSON(wd.InsertData)
util.KillPipelineIfErr(err, killChan)
err = util.SQLInsertData(s.writeDB, dd, wd.TableName, s.OnDupKeyUpdate, s.OnDupKeyFields, s.BatchSize)
util.KillPipelineIfErr(err, killChan)
} else {
logger.Debug("SQLWriter: normal data scenario")
err = util.SQLInsertData(s.writeDB, d, s.TableName, s.OnDupKeyUpdate, s.OnDupKeyFields, s.BatchSize)
util.KillPipelineIfErr(err, killChan)
}
logger.Info("SQLWriter: Write complete")
}
func ExampleNewJSON() {
t := testStruct{A: 1, B: 2}
d, _ := data.NewJSON(t)
fmt.Println(string(d))
// Output: {"A":1,"B":2}
}
func sendTableData(tableData []map[string]interface{}, dataChan chan data.JSON) {
d, err := data.NewJSON(tableData)
if err != nil {
sendErr(err, dataChan)
} else {
dataChan <- d
}
}
func scanRowsUsingStruct(rows *sql.Rows, columns []string, structDest interface{}, batchSize int, dataChan chan data.JSON) {
defer rows.Close()
tableData := []map[string]interface{}{}
for rows.Next() {
err := sqlstruct.Scan(structDest, rows)
if err != nil {
sendErr(err, dataChan)
}
d, err := data.NewJSON(structDest)
if err != nil {
sendErr(err, dataChan)
}
entry := make(map[string]interface{})
err = data.ParseJSON(d, &entry)
if err != nil {
sendErr(err, dataChan)
}
tableData = append(tableData, entry)
if batchSize > 0 && len(tableData) >= batchSize {
sendTableData(tableData, dataChan)
tableData = []map[string]interface{}{}
}
}
if rows.Err() != nil {
sendErr(rows.Err(), dataChan)
}
// Flush remaining tableData
if len(tableData) > 0 {
sendTableData(tableData, dataChan)
}
close(dataChan) // signal completion to caller
}
func (r *SftpReader) sendFilePath(path string, outputChan chan data.JSON, killChan chan error) {
sftpPath := util.SftpPath{Path: path}
d, err := data.NewJSON(sftpPath)
util.KillPipelineIfErr(err, killChan)
outputChan <- d
}