// ForEachQueryData handles generating the SQL (in case of dynamic mode),
// running the query and retrieving the data in data.JSON format, and then
// passing the results back witih the function call to forEach.
func (r *BigQueryReader) ForEachQueryData(d data.JSON, killChan chan error, forEach func(d data.JSON)) {
sql := ""
var err error
if r.query == "" && r.sqlGenerator != nil {
sql, err = r.sqlGenerator(d)
util.KillPipelineIfErr(err, killChan)
} else if r.query != "" {
sql = r.query
} else {
killChan <- errors.New("BigQueryReader: must have either static query or sqlGenerator func")
}
logger.Debug("BigQueryReader: Running -", sql)
bqDataChan := make(chan bigquery.Data)
go r.bqClient().AsyncQuery(r.PageSize, r.config.DatasetID, r.config.ProjectID, sql, bqDataChan)
aggregatedData := bigquery.Data{}
for bqd := range bqDataChan {
util.KillPipelineIfErr(bqd.Err, killChan)
logger.Info("BigQueryReader: received bqData: len(rows) =", len(bqd.Rows))
// logger.Debug(" %+v", bqd)
if bqd.Rows != nil && bqd.Headers != nil && len(bqd.Rows) > 0 {
if r.AggregateResults {
logger.Debug("BigQueryReader: aggregating results")
aggregatedData.Headers = bqd.Headers
aggregatedData.Rows = append(aggregatedData.Rows, bqd.Rows...)
} else {
// Send data as soon as we get it back
logger.Debug("BigQueryReader: sending data without aggregation")
d, err := data.JSONFromHeaderAndRows(bqd.Headers, bqd.Rows)
util.KillPipelineIfErr(err, killChan)
forEach(d) // pass back out via the forEach func
}
}
}
if r.AggregateResults {
logger.Info("BigQueryReader: sending aggregated results: len(rows) =", len(aggregatedData.Rows))
d, err := data.JSONFromHeaderAndRows(aggregatedData.Headers, aggregatedData.Rows)
util.KillPipelineIfErr(err, killChan)
forEach(d) // pass back out via the forEach func
}
}
func ExampleJSONFromHeaderAndRows() {
header := []string{"A", "B", "C"}
rows := [][]interface{}{
[]interface{}{1, 2, 3},
[]interface{}{4, 5, 6},
}
d, _ := data.JSONFromHeaderAndRows(header, rows)
fmt.Println(fmt.Sprintf("%+v", string(d)))
// Output: [{"A":1,"B":2,"C":3},{"A":4,"B":5,"C":6}]
}