Esempio n. 1
0
func New(client *http.Client, projectId, datasetId string) (result *BigQuery, err error) {
	service, err := gbigquery.New(client)
	if err != nil {
		return
	}
	result = &BigQuery{
		service:   service,
		projectId: projectId,
		datasetId: datasetId,
	}
	return
}
Esempio n. 2
0
// Helper method to create an authenticated connection.
func connect() (*oauth2.Token, *bigquery.Service, error) {
	if *clientId == "" {
		return nil, nil, fmt.Errorf("no client id specified")
	}
	if *serviceAccount == "" {
		return nil, nil, fmt.Errorf("no service account specified")
	}
	if *projectId == "" {
		return nil, nil, fmt.Errorf("no project id specified")
	}
	authScope := bigquery.BigqueryScope
	if *pemFile == "" {
		return nil, nil, fmt.Errorf("no credentials specified")
	}
	pemBytes, err := ioutil.ReadFile(*pemFile)
	if err != nil {
		return nil, nil, fmt.Errorf("could not access credential file %v - %v", pemFile, err)
	}

	jwtConfig := &jwt.Config{
		Email:      *serviceAccount,
		Scopes:     []string{authScope},
		PrivateKey: pemBytes,
		TokenURL:   "https://accounts.google.com/o/oauth2/token",
	}
	token, err := jwtConfig.TokenSource(oauth2.NoContext).Token()
	if err != nil {
		return nil, nil, err
	}
	if !token.Valid() {
		return nil, nil, fmt.Errorf("invalid token for BigQuery oauth")
	}

	config := &oauth2.Config{
		ClientID:     *clientId,
		ClientSecret: *clientSecret,
		Scopes:       []string{authScope},
		Endpoint: oauth2.Endpoint{
			AuthURL:  "https://accounts.google.com/o/oauth2/auth",
			TokenURL: "https://accounts.google.com/o/oauth2/token",
		},
	}
	client := config.Client(oauth2.NoContext, token)

	service, err := bigquery.New(client)
	if err != nil {
		fmt.Printf("Failed to create new service: %v\n", err)
		return nil, nil, err
	}

	return token, service, nil
}
Esempio n. 3
0
// Helper method to create an authenticated connection.
func connect() (*oauth.Token, *bigquery.Service, error) {
	if *clientId == "" {
		return nil, nil, fmt.Errorf("no client id specified")
	}
	if *serviceAccount == "" {
		return nil, nil, fmt.Errorf("no service account specified")
	}
	if *projectId == "" {
		return nil, nil, fmt.Errorf("no project id specified")
	}
	authScope := bigquery.BigqueryScope
	if *pemFile == "" {
		return nil, nil, fmt.Errorf("no credentials specified")
	}
	pemBytes, err := ioutil.ReadFile(*pemFile)
	if err != nil {
		return nil, nil, fmt.Errorf("could not access credential file %v - %v", pemFile, err)
	}

	t := jwt.NewToken(*serviceAccount, authScope, pemBytes)
	token, err := t.Assert(&http.Client{})
	if err != nil {
		fmt.Printf("Invalid token: %v\n", err)
		return nil, nil, err
	}
	config := &oauth.Config{
		ClientId:     *clientId,
		ClientSecret: *clientSecret,
		Scope:        authScope,
		AuthURL:      "https://accounts.google.com/o/oauth2/auth",
		TokenURL:     "https://accounts.google.com/o/oauth2/token",
	}

	transport := &oauth.Transport{
		Token:  token,
		Config: config,
	}
	client := transport.Client()

	service, err := bigquery.New(client)
	if err != nil {
		fmt.Printf("Failed to create new service: %v\n", err)
		return nil, nil, err
	}

	return token, service, nil
}
Esempio n. 4
0
// NewClient ...
func NewClient(email, pemKeyPath string) (*Client, error) {
	pemKeyBytes, err := ioutil.ReadFile(pemKeyPath)
	if err != nil {
		return nil, err
	}
	conf := jwt.Config{
		Email:      email,
		PrivateKey: pemKeyBytes,
		Scopes:     []string{bigquery.BigqueryScope},
		TokenURL:   google.JWTTokenURL,
	}
	client := conf.Client(oauth2.NoContext)
	service, err := bigquery.New(client)
	if err != nil {
		return nil, err
	}
	return &Client{BigQueryService: service}, nil
}
Esempio n. 5
0
func newBQDataset(client *http.Client, dsProj string, dsId string) (*bqDataset,
	error) {

	service, err := bigquery.New(client)
	if err != nil {
		return nil, err
	}

	return &bqDataset{
		project: dsProj,
		id:      dsId,
		bq:      service,
		dataset: &bigquery.Dataset{
			DatasetReference: &bigquery.DatasetReference{
				DatasetId: dsId,
				ProjectId: dsProj,
			},
		},
		jobsets: make(map[string]*list.List),
	}, nil
}
Esempio n. 6
0
func queryIntoDatastore(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {

	limitUpper := util.MonthsBack(1)
	limitLower := util.MonthsBack(25)

	var q bq.QueryRequest = bq.QueryRequest{}
	q.Query = `
		SELECT
		  repository_language
		, LEFT(repository_pushed_at,7) monthx
		, CEIL( count(*)/1000) Tausend
		FROM githubarchive:github.timeline
		where 1=1
			AND  LEFT(repository_pushed_at,7) >= '` + limitLower + `'
			AND  LEFT(repository_pushed_at,7) <= '` + limitUpper + `'
			AND  repository_language in ('Go','go','Golang','golang','C','Java','PHP','JavaScript','C++','Python','Ruby')
			AND  type="PushEvent"
		group by monthx, repository_language
		order by repository_language   , monthx
		;
	`

	c := appengine.NewContext(r)
	config := oauth2_google.NewAppEngineConfig(c, []string{
		"https://www.googleapis.com/auth/bigquery",
	})
	// The following client will be authorized by the App Engine
	// app's service account for the provided scopes.
	client := http.Client{Transport: config.NewTransport()}
	//client.Get("...")

	//oauthHttpClient := &http.Client{}
	bigqueryService, err := bq.New(&client)
	util_err.Err_http(w, r, err, false)

	fmt.Fprint(w, "s1<br>\n")

	// Create a query statement and query request object
	//  query_data = {'query':'SELECT TOP(title, 10) as title, COUNT(*) as revision_count FROM [publicdata:samples.wikipedia] WHERE wp_namespace = 0;'}
	//  query_request = bigquery_service.jobs()
	// Make a call to the BigQuery API
	//  query_response = query_request.query(projectId=PROJECT_NUMBER, body=query_data).execute()

	js := bq.NewJobsService(bigqueryService)
	jqc := js.Query("347979071940", &q)

	fmt.Fprint(w, "s2 "+util.TimeMarker()+" <br>\n")
	resp, err := jqc.Do()
	util_err.Err_http(w, r, err, false)

	rows := resp.Rows
	var vVDest [][]byte = make([][]byte, len(rows))

	c.Errorf("%#v", rows)

	for i0, v0 := range rows {

		cells := v0.F

		b_row := new(bytes.Buffer)
		b_row.WriteString(fmt.Sprintf("r%0.2d -- ", i0))
		for i1, v1 := range cells {
			val1 := v1.V
			b_row.WriteString(fmt.Sprintf("c%0.2d: %v  ", i1, val1))
		}
		vVDest[i0] = []byte(b_row.Bytes())
	}

	key_combi, _ := dsu.BufPut(w, r, dsu.WrapBlob{Name: "bq_res1", VVByte: vVDest}, "bq_res1")
	dsObj, _ := dsu.BufGet(w, r, key_combi)

	printPlaintextTable(w, r, dsObj.VVByte)

	fmt.Fprint(w, "s3 "+util.TimeMarker()+" <br>\n")

}