// largeDataPagedQuery builds a job and inserts it into the job queue allowing the flexibility to set the custom AllowLargeResults flag for the job func (c *Client) largeDataPagedQuery(service *bigquery.Service, pageSize int, dataset, project, queryStr string, dataChan chan Data) ([][]interface{}, []string, error) { c.printDebug("largeDataPagedQuery starting") ts := time.Now() // start query tableRef := bigquery.TableReference{DatasetId: dataset, ProjectId: project, TableId: c.tempTableName} jobConfigQuery := bigquery.JobConfigurationQuery{} datasetRef := &bigquery.DatasetReference{ DatasetId: dataset, ProjectId: project, } jobConfigQuery.AllowLargeResults = true jobConfigQuery.Query = queryStr jobConfigQuery.DestinationTable = &tableRef jobConfigQuery.DefaultDataset = datasetRef if !c.flattenResults { c.printDebug("setting FlattenResults to false") // need a pointer to bool f := false jobConfigQuery.FlattenResults = &f } jobConfigQuery.WriteDisposition = "WRITE_TRUNCATE" jobConfigQuery.CreateDisposition = "CREATE_IF_NEEDED" jobConfig := bigquery.JobConfiguration{} jobConfig.Query = &jobConfigQuery job := bigquery.Job{} job.Configuration = &jobConfig jobInsert := service.Jobs.Insert(project, &job) runningJob, jerr := jobInsert.Do() if jerr != nil { c.printDebug("Error inserting job!", jerr) if dataChan != nil { dataChan <- Data{Err: jerr} } return nil, nil, jerr } qr, err := service.Jobs.GetQueryResults(project, runningJob.JobReference.JobId).Do() if err != nil { c.printDebug("Error loading query: ", err) if dataChan != nil { dataChan <- Data{Err: err} } return nil, nil, err } var headers []string rows := [][]interface{}{} // if query is completed process, otherwise begin checking for results if qr.JobComplete { c.printDebug("job complete, got rows", len(qr.Rows)) headers, rows = c.headersAndRows(qr.Schema, qr.Rows) if dataChan != nil { dataChan <- Data{Headers: headers, Rows: rows} } } if !qr.JobComplete { resultChan := make(chan [][]interface{}) headersChan := make(chan []string) go c.pageOverJob(len(rows), runningJob.JobReference, qr.PageToken, resultChan, headersChan) L: for { select { case h, ok := <-headersChan: if ok { c.printDebug("got headers") headers = h } case newRows, ok := <-resultChan: if !ok { break L } if dataChan != nil { c.printDebug("got rows", len(newRows)) dataChan <- Data{Headers: headers, Rows: newRows} } else { rows = append(rows, newRows...) } } } } if dataChan != nil { close(dataChan) } c.printDebug("largeDataPagedQuery completed in ", time.Now().Sub(ts).Seconds(), "s") return rows, headers, nil }