// insertTableToBigQuery inserts a single table to BigQuery using BigQuery's InsertAll request. // // This function is assigned to Streamer.insertTable member. // It is overridable so we can test Streamer without actually inserting anythin to BigQuery. func (b *Streamer) insertTableToBigQuery(projectID, datasetID, tableID string, t table) ( r *bigquery.TableDataInsertAllResponse, err error) { // Convert all rows to bigquery table rows. rows := make([]*bigquery.TableDataInsertAllRequestRows, len(t)) for i, row := range t { rows[i] = &bigquery.TableDataInsertAllRequestRows{ InsertId: row.rowID, Json: row.jsonValue} } // Generate request, tabledata and send. request := bigquery.TableDataInsertAllRequest{Kind: "bigquery#tableDataInsertAllRequest", Rows: rows} // TODO might be better to cache table services somehow, instead of re-creating them on every flush. tableService := bigquery.NewTabledataService(b.service) r, err = tableService.InsertAll(projectID, datasetID, tableID, &request).Do() return }
func newTableStreamer(streamer *Streamer, table string, suffix func() string) *tableStreamer { ts := &tableStreamer{ streamer: streamer, service: bigquery.NewTabledataService(streamer.service), table: table, suffix: suffix, incoming: make(chan interface{}, bufferSize), stop: make(chan struct{}), lastID: rand.Int63(), flushInterval: 10 * time.Second, flushMax: bufferSize, crankiness: backoff.NewExponentialBackOff(), } ts.crankiness.MaxElapsedTime = 0 ts.crankiness.InitialInterval = 2 * time.Second ts.crankiness.NextBackOff() return ts }