func (ds *bqDataset) Search(params *CampaignParams) (*[]AppBuffer, error) {
data := map[string]interface{}{
"LIMIT": SEARCH_APPS_LIMIT,
"DATASET": BIGQUERY_DATASET,
"TABLE_DATA": BIGQUERY_TABLE_DATA,
"TABLE_PROCEED": getTableProceed(),
}
t := template.Must(template.New("email").Parse(queryNewAppsTmpl))
buf := &bytes.Buffer{}
if err := t.Execute(buf, data); err != nil {
return nil, err
}
queryRequest := &bigquery.QueryRequest{
//DefaultDataset: datasetRef,
Query: buf.String(),
MaxResults: SEARCH_APPS_LIMIT, //int64(max),
Kind: "json",
//Kind: "igquery#queryRequest",
TimeoutMs: BIGQUERY_TIMEOUT,
}
jobsService := bigquery.NewJobsService(ds.bq)
queryResponse, err := jobsService.Query(BIGQUERY_PROJECT, queryRequest).Do()
if err != nil {
return nil, err
}
//c.Debugf(strconv.Itoa(len(queryResponse.Rows)))
var appsBuffer []AppBuffer
err = toStructs(queryResponse, &appsBuffer)
if err != nil {
return nil, err
}
return &appsBuffer, nil
}
func queryIntoDatastore(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {
limitUpper := util.MonthsBack(1)
limitLower := util.MonthsBack(25)
var q bq.QueryRequest = bq.QueryRequest{}
q.Query = `
SELECT
repository_language
, LEFT(repository_pushed_at,7) monthx
, CEIL( count(*)/1000) Tausend
FROM githubarchive:github.timeline
where 1=1
AND LEFT(repository_pushed_at,7) >= '` + limitLower + `'
AND LEFT(repository_pushed_at,7) <= '` + limitUpper + `'
AND repository_language in ('Go','go','Golang','golang','C','Java','PHP','JavaScript','C++','Python','Ruby')
AND type="PushEvent"
group by monthx, repository_language
order by repository_language , monthx
;
`
c := appengine.NewContext(r)
// The following client will be authorized by the App Engine
// app's service account for the provided scopes.
// "https://www.googleapis.com/auth/bigquery"
// "https://www.googleapis.com/auth/devstorage.full_control"
// 2015-06: instead of oauth2.NoContext we get a new type of context
var ctx context.Context = appengine.NewContext(r)
oauthHttpClient, err := google.DefaultClient(
ctx, "https://www.googleapis.com/auth/bigquery")
if err != nil {
log.Fatal(err)
}
bigqueryService, err := bq.New(oauthHttpClient)
loghttp.E(w, r, err, false)
fmt.Fprint(w, "s1<br>\n")
// Create a query statement and query request object
// query_data = {'query':'SELECT TOP(title, 10) as title, COUNT(*) as revision_count FROM [publicdata:samples.wikipedia] WHERE wp_namespace = 0;'}
// query_request = bigquery_service.jobs()
// Make a call to the BigQuery API
// query_response = query_request.query(projectId=PROJECT_NUMBER, body=query_data).execute()
js := bq.NewJobsService(bigqueryService)
jqc := js.Query("347979071940", &q)
fmt.Fprint(w, "s2 "+util.TimeMarker()+" <br>\n")
resp, err := jqc.Do()
loghttp.E(w, r, err, false)
rows := resp.Rows
var vVDest [][]byte = make([][]byte, len(rows))
aelog.Errorf(c, "%#v", rows)
for i0, v0 := range rows {
cells := v0.F
b_row := new(bytes.Buffer)
b_row.WriteString(fmt.Sprintf("r%0.2d -- ", i0))
for i1, v1 := range cells {
val1 := v1.V
b_row.WriteString(fmt.Sprintf("c%0.2d: %v ", i1, val1))
}
vVDest[i0] = []byte(b_row.Bytes())
}
key_combi, _ := dsu.BufPut(c, dsu.WrapBlob{Name: "bq_res1", VVByte: vVDest}, "bq_res1")
dsObj, _ := dsu.BufGet(c, key_combi)
printPlaintextTable(w, r, dsObj.VVByte)
fmt.Fprint(w, "s3 "+util.TimeMarker()+" <br>\n")
}