func New(client *http.Client, projectId, datasetId string) (result *BigQuery, err error) {
service, err := gbigquery.New(client)
if err != nil {
return
}
result = &BigQuery{
service: service,
projectId: projectId,
datasetId: datasetId,
}
return
}
// Helper method to create an authenticated connection.
func connect() (*oauth2.Token, *bigquery.Service, error) {
if *clientId == "" {
return nil, nil, fmt.Errorf("no client id specified")
}
if *serviceAccount == "" {
return nil, nil, fmt.Errorf("no service account specified")
}
if *projectId == "" {
return nil, nil, fmt.Errorf("no project id specified")
}
authScope := bigquery.BigqueryScope
if *pemFile == "" {
return nil, nil, fmt.Errorf("no credentials specified")
}
pemBytes, err := ioutil.ReadFile(*pemFile)
if err != nil {
return nil, nil, fmt.Errorf("could not access credential file %v - %v", pemFile, err)
}
jwtConfig := &jwt.Config{
Email: *serviceAccount,
Scopes: []string{authScope},
PrivateKey: pemBytes,
TokenURL: "https://accounts.google.com/o/oauth2/token",
}
token, err := jwtConfig.TokenSource(oauth2.NoContext).Token()
if err != nil {
return nil, nil, err
}
if !token.Valid() {
return nil, nil, fmt.Errorf("invalid token for BigQuery oauth")
}
config := &oauth2.Config{
ClientID: *clientId,
ClientSecret: *clientSecret,
Scopes: []string{authScope},
Endpoint: oauth2.Endpoint{
AuthURL: "https://accounts.google.com/o/oauth2/auth",
TokenURL: "https://accounts.google.com/o/oauth2/token",
},
}
client := config.Client(oauth2.NoContext, token)
service, err := bigquery.New(client)
if err != nil {
fmt.Printf("Failed to create new service: %v\n", err)
return nil, nil, err
}
return token, service, nil
}
// Helper method to create an authenticated connection.
func connect() (*oauth.Token, *bigquery.Service, error) {
if *clientId == "" {
return nil, nil, fmt.Errorf("no client id specified")
}
if *serviceAccount == "" {
return nil, nil, fmt.Errorf("no service account specified")
}
if *projectId == "" {
return nil, nil, fmt.Errorf("no project id specified")
}
authScope := bigquery.BigqueryScope
if *pemFile == "" {
return nil, nil, fmt.Errorf("no credentials specified")
}
pemBytes, err := ioutil.ReadFile(*pemFile)
if err != nil {
return nil, nil, fmt.Errorf("could not access credential file %v - %v", pemFile, err)
}
t := jwt.NewToken(*serviceAccount, authScope, pemBytes)
token, err := t.Assert(&http.Client{})
if err != nil {
fmt.Printf("Invalid token: %v\n", err)
return nil, nil, err
}
config := &oauth.Config{
ClientId: *clientId,
ClientSecret: *clientSecret,
Scope: authScope,
AuthURL: "https://accounts.google.com/o/oauth2/auth",
TokenURL: "https://accounts.google.com/o/oauth2/token",
}
transport := &oauth.Transport{
Token: token,
Config: config,
}
client := transport.Client()
service, err := bigquery.New(client)
if err != nil {
fmt.Printf("Failed to create new service: %v\n", err)
return nil, nil, err
}
return token, service, nil
}
// NewClient ...
func NewClient(email, pemKeyPath string) (*Client, error) {
pemKeyBytes, err := ioutil.ReadFile(pemKeyPath)
if err != nil {
return nil, err
}
conf := jwt.Config{
Email: email,
PrivateKey: pemKeyBytes,
Scopes: []string{bigquery.BigqueryScope},
TokenURL: google.JWTTokenURL,
}
client := conf.Client(oauth2.NoContext)
service, err := bigquery.New(client)
if err != nil {
return nil, err
}
return &Client{BigQueryService: service}, nil
}
func newBQDataset(client *http.Client, dsProj string, dsId string) (*bqDataset,
error) {
service, err := bigquery.New(client)
if err != nil {
return nil, err
}
return &bqDataset{
project: dsProj,
id: dsId,
bq: service,
dataset: &bigquery.Dataset{
DatasetReference: &bigquery.DatasetReference{
DatasetId: dsId,
ProjectId: dsProj,
},
},
jobsets: make(map[string]*list.List),
}, nil
}
func queryIntoDatastore(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {
limitUpper := util.MonthsBack(1)
limitLower := util.MonthsBack(25)
var q bq.QueryRequest = bq.QueryRequest{}
q.Query = `
SELECT
repository_language
, LEFT(repository_pushed_at,7) monthx
, CEIL( count(*)/1000) Tausend
FROM githubarchive:github.timeline
where 1=1
AND LEFT(repository_pushed_at,7) >= '` + limitLower + `'
AND LEFT(repository_pushed_at,7) <= '` + limitUpper + `'
AND repository_language in ('Go','go','Golang','golang','C','Java','PHP','JavaScript','C++','Python','Ruby')
AND type="PushEvent"
group by monthx, repository_language
order by repository_language , monthx
;
`
c := appengine.NewContext(r)
config := oauth2_google.NewAppEngineConfig(c, []string{
"https://www.googleapis.com/auth/bigquery",
})
// The following client will be authorized by the App Engine
// app's service account for the provided scopes.
client := http.Client{Transport: config.NewTransport()}
//client.Get("...")
//oauthHttpClient := &http.Client{}
bigqueryService, err := bq.New(&client)
util_err.Err_http(w, r, err, false)
fmt.Fprint(w, "s1<br>\n")
// Create a query statement and query request object
// query_data = {'query':'SELECT TOP(title, 10) as title, COUNT(*) as revision_count FROM [publicdata:samples.wikipedia] WHERE wp_namespace = 0;'}
// query_request = bigquery_service.jobs()
// Make a call to the BigQuery API
// query_response = query_request.query(projectId=PROJECT_NUMBER, body=query_data).execute()
js := bq.NewJobsService(bigqueryService)
jqc := js.Query("347979071940", &q)
fmt.Fprint(w, "s2 "+util.TimeMarker()+" <br>\n")
resp, err := jqc.Do()
util_err.Err_http(w, r, err, false)
rows := resp.Rows
var vVDest [][]byte = make([][]byte, len(rows))
c.Errorf("%#v", rows)
for i0, v0 := range rows {
cells := v0.F
b_row := new(bytes.Buffer)
b_row.WriteString(fmt.Sprintf("r%0.2d -- ", i0))
for i1, v1 := range cells {
val1 := v1.V
b_row.WriteString(fmt.Sprintf("c%0.2d: %v ", i1, val1))
}
vVDest[i0] = []byte(b_row.Bytes())
}
key_combi, _ := dsu.BufPut(w, r, dsu.WrapBlob{Name: "bq_res1", VVByte: vVDest}, "bq_res1")
dsObj, _ := dsu.BufGet(w, r, key_combi)
printPlaintextTable(w, r, dsObj.VVByte)
fmt.Fprint(w, "s3 "+util.TimeMarker()+" <br>\n")
}