// ProcessData defers to util.SQLInsertData
func (s *SQLWriter) ProcessData(d data.JSON, outputChan chan data.JSON, killChan chan error) {
// handle panics a bit more gracefully
defer func() {
if err := recover(); err != nil {
util.KillPipelineIfErr(err.(error), killChan)
}
}()
// First check for SQLWriterData
var wd SQLWriterData
err := data.ParseJSONSilent(d, &wd)
logger.Info("SQLWriter: Writing data...")
if err == nil && wd.TableName != "" && wd.InsertData != nil {
logger.Debug("SQLWriter: SQLWriterData scenario")
dd, err := data.NewJSON(wd.InsertData)
util.KillPipelineIfErr(err, killChan)
err = util.SQLInsertData(s.writeDB, dd, wd.TableName, s.OnDupKeyUpdate, s.OnDupKeyFields, s.BatchSize)
util.KillPipelineIfErr(err, killChan)
} else {
logger.Debug("SQLWriter: normal data scenario")
err = util.SQLInsertData(s.writeDB, d, s.TableName, s.OnDupKeyUpdate, s.OnDupKeyFields, s.BatchSize)
util.KillPipelineIfErr(err, killChan)
}
logger.Info("SQLWriter: Write complete")
}
// ForEachQueryData handles generating the SQL (in case of dynamic mode),
// running the query and retrieving the data in data.JSON format, and then
// passing the results back witih the function call to forEach.
func (s *SQLReader) ForEachQueryData(d data.JSON, killChan chan error, forEach func(d data.JSON)) {
sql := ""
var err error
if s.query == "" && s.sqlGenerator != nil {
sql, err = s.sqlGenerator(d)
util.KillPipelineIfErr(err, killChan)
} else if s.query != "" {
sql = s.query
} else {
killChan <- errors.New("SQLReader: must have either static query or sqlGenerator func")
}
logger.Debug("SQLReader: Running - ", sql)
// See sql.go
dataChan, err := util.GetDataFromSQLQuery(s.readDB, sql, s.BatchSize, s.StructDestination)
util.KillPipelineIfErr(err, killChan)
for d := range dataChan {
// First check if an error was returned back from the SQL processing
// helper, then if not call forEach with the received data.
var derr dataErr
if err := data.ParseJSONSilent(d, &derr); err == nil {
util.KillPipelineIfErr(errors.New(derr.Error), killChan)
} else {
forEach(d)
}
}
}
// http://play.golang.org/p/2wHfO6YS3_
func determineBytesValue(b []byte) (interface{}, error) {
var v interface{}
err := data.ParseJSONSilent(b, &v)
if err != nil {
// need to quote strings for JSON to parse correctly
if !strings.Contains(string(b), `"`) {
b = []byte(fmt.Sprintf(`"%v"`, string(b)))
return determineBytesValue(b)
}
}
switch vv := v.(type) {
case []byte:
return string(vv), err
default:
return v, err
}
}