func New(client *http.Client, projectId, datasetId string) (result *BigQuery, err error) { service, err := gbigquery.New(client) if err != nil { return } result = &BigQuery{ service: service, projectId: projectId, datasetId: datasetId, } return }
// Helper method to create an authenticated connection. func connect() (*oauth2.Token, *bigquery.Service, error) { if *clientId == "" { return nil, nil, fmt.Errorf("no client id specified") } if *serviceAccount == "" { return nil, nil, fmt.Errorf("no service account specified") } if *projectId == "" { return nil, nil, fmt.Errorf("no project id specified") } authScope := bigquery.BigqueryScope if *pemFile == "" { return nil, nil, fmt.Errorf("no credentials specified") } pemBytes, err := ioutil.ReadFile(*pemFile) if err != nil { return nil, nil, fmt.Errorf("could not access credential file %v - %v", pemFile, err) } jwtConfig := &jwt.Config{ Email: *serviceAccount, Scopes: []string{authScope}, PrivateKey: pemBytes, TokenURL: "https://accounts.google.com/o/oauth2/token", } token, err := jwtConfig.TokenSource(oauth2.NoContext).Token() if err != nil { return nil, nil, err } if !token.Valid() { return nil, nil, fmt.Errorf("invalid token for BigQuery oauth") } config := &oauth2.Config{ ClientID: *clientId, ClientSecret: *clientSecret, Scopes: []string{authScope}, Endpoint: oauth2.Endpoint{ AuthURL: "https://accounts.google.com/o/oauth2/auth", TokenURL: "https://accounts.google.com/o/oauth2/token", }, } client := config.Client(oauth2.NoContext, token) service, err := bigquery.New(client) if err != nil { fmt.Printf("Failed to create new service: %v\n", err) return nil, nil, err } return token, service, nil }
// Helper method to create an authenticated connection. func connect() (*oauth.Token, *bigquery.Service, error) { if *clientId == "" { return nil, nil, fmt.Errorf("no client id specified") } if *serviceAccount == "" { return nil, nil, fmt.Errorf("no service account specified") } if *projectId == "" { return nil, nil, fmt.Errorf("no project id specified") } authScope := bigquery.BigqueryScope if *pemFile == "" { return nil, nil, fmt.Errorf("no credentials specified") } pemBytes, err := ioutil.ReadFile(*pemFile) if err != nil { return nil, nil, fmt.Errorf("could not access credential file %v - %v", pemFile, err) } t := jwt.NewToken(*serviceAccount, authScope, pemBytes) token, err := t.Assert(&http.Client{}) if err != nil { fmt.Printf("Invalid token: %v\n", err) return nil, nil, err } config := &oauth.Config{ ClientId: *clientId, ClientSecret: *clientSecret, Scope: authScope, AuthURL: "https://accounts.google.com/o/oauth2/auth", TokenURL: "https://accounts.google.com/o/oauth2/token", } transport := &oauth.Transport{ Token: token, Config: config, } client := transport.Client() service, err := bigquery.New(client) if err != nil { fmt.Printf("Failed to create new service: %v\n", err) return nil, nil, err } return token, service, nil }
// NewClient ... func NewClient(email, pemKeyPath string) (*Client, error) { pemKeyBytes, err := ioutil.ReadFile(pemKeyPath) if err != nil { return nil, err } conf := jwt.Config{ Email: email, PrivateKey: pemKeyBytes, Scopes: []string{bigquery.BigqueryScope}, TokenURL: google.JWTTokenURL, } client := conf.Client(oauth2.NoContext) service, err := bigquery.New(client) if err != nil { return nil, err } return &Client{BigQueryService: service}, nil }
func newBQDataset(client *http.Client, dsProj string, dsId string) (*bqDataset, error) { service, err := bigquery.New(client) if err != nil { return nil, err } return &bqDataset{ project: dsProj, id: dsId, bq: service, dataset: &bigquery.Dataset{ DatasetReference: &bigquery.DatasetReference{ DatasetId: dsId, ProjectId: dsProj, }, }, jobsets: make(map[string]*list.List), }, nil }
func queryIntoDatastore(w http.ResponseWriter, r *http.Request, m map[string]interface{}) { limitUpper := util.MonthsBack(1) limitLower := util.MonthsBack(25) var q bq.QueryRequest = bq.QueryRequest{} q.Query = ` SELECT repository_language , LEFT(repository_pushed_at,7) monthx , CEIL( count(*)/1000) Tausend FROM githubarchive:github.timeline where 1=1 AND LEFT(repository_pushed_at,7) >= '` + limitLower + `' AND LEFT(repository_pushed_at,7) <= '` + limitUpper + `' AND repository_language in ('Go','go','Golang','golang','C','Java','PHP','JavaScript','C++','Python','Ruby') AND type="PushEvent" group by monthx, repository_language order by repository_language , monthx ; ` c := appengine.NewContext(r) config := oauth2_google.NewAppEngineConfig(c, []string{ "https://www.googleapis.com/auth/bigquery", }) // The following client will be authorized by the App Engine // app's service account for the provided scopes. client := http.Client{Transport: config.NewTransport()} //client.Get("...") //oauthHttpClient := &http.Client{} bigqueryService, err := bq.New(&client) util_err.Err_http(w, r, err, false) fmt.Fprint(w, "s1<br>\n") // Create a query statement and query request object // query_data = {'query':'SELECT TOP(title, 10) as title, COUNT(*) as revision_count FROM [publicdata:samples.wikipedia] WHERE wp_namespace = 0;'} // query_request = bigquery_service.jobs() // Make a call to the BigQuery API // query_response = query_request.query(projectId=PROJECT_NUMBER, body=query_data).execute() js := bq.NewJobsService(bigqueryService) jqc := js.Query("347979071940", &q) fmt.Fprint(w, "s2 "+util.TimeMarker()+" <br>\n") resp, err := jqc.Do() util_err.Err_http(w, r, err, false) rows := resp.Rows var vVDest [][]byte = make([][]byte, len(rows)) c.Errorf("%#v", rows) for i0, v0 := range rows { cells := v0.F b_row := new(bytes.Buffer) b_row.WriteString(fmt.Sprintf("r%0.2d -- ", i0)) for i1, v1 := range cells { val1 := v1.V b_row.WriteString(fmt.Sprintf("c%0.2d: %v ", i1, val1)) } vVDest[i0] = []byte(b_row.Bytes()) } key_combi, _ := dsu.BufPut(w, r, dsu.WrapBlob{Name: "bq_res1", VVByte: vVDest}, "bq_res1") dsObj, _ := dsu.BufGet(w, r, key_combi) printPlaintextTable(w, r, dsObj.VVByte) fmt.Fprint(w, "s3 "+util.TimeMarker()+" <br>\n") }