// ForEachQueryData handles generating the SQL (in case of dynamic mode), // running the query and retrieving the data in data.JSON format, and then // passing the results back witih the function call to forEach. func (r *BigQueryReader) ForEachQueryData(d data.JSON, killChan chan error, forEach func(d data.JSON)) { sql := "" var err error if r.query == "" && r.sqlGenerator != nil { sql, err = r.sqlGenerator(d) util.KillPipelineIfErr(err, killChan) } else if r.query != "" { sql = r.query } else { killChan <- errors.New("BigQueryReader: must have either static query or sqlGenerator func") } logger.Debug("BigQueryReader: Running -", sql) bqDataChan := make(chan bigquery.Data) go r.bqClient().AsyncQuery(r.PageSize, r.config.DatasetID, r.config.ProjectID, sql, bqDataChan) aggregatedData := bigquery.Data{} for bqd := range bqDataChan { util.KillPipelineIfErr(bqd.Err, killChan) logger.Info("BigQueryReader: received bqData: len(rows) =", len(bqd.Rows)) // logger.Debug(" %+v", bqd) if bqd.Rows != nil && bqd.Headers != nil && len(bqd.Rows) > 0 { if r.AggregateResults { logger.Debug("BigQueryReader: aggregating results") aggregatedData.Headers = bqd.Headers aggregatedData.Rows = append(aggregatedData.Rows, bqd.Rows...) } else { // Send data as soon as we get it back logger.Debug("BigQueryReader: sending data without aggregation") d, err := data.JSONFromHeaderAndRows(bqd.Headers, bqd.Rows) util.KillPipelineIfErr(err, killChan) forEach(d) // pass back out via the forEach func } } } if r.AggregateResults { logger.Info("BigQueryReader: sending aggregated results: len(rows) =", len(aggregatedData.Rows)) d, err := data.JSONFromHeaderAndRows(aggregatedData.Headers, aggregatedData.Rows) util.KillPipelineIfErr(err, killChan) forEach(d) // pass back out via the forEach func } }
func ExampleJSONFromHeaderAndRows() { header := []string{"A", "B", "C"} rows := [][]interface{}{ []interface{}{1, 2, 3}, []interface{}{4, 5, 6}, } d, _ := data.JSONFromHeaderAndRows(header, rows) fmt.Println(fmt.Sprintf("%+v", string(d))) // Output: [{"A":1,"B":2,"C":3},{"A":4,"B":5,"C":6}] }