/*
Create view of a table defined by a query.
*/
func (self *BigQuery) AssertView(viewName string, query string) (err error) {
tablesService := gbigquery.NewTablesService(self.service)
_, err = tablesService.Get(self.projectId, self.datasetId, viewName).Do()
if err != nil {
if gapiErr, ok := err.(*googleapi.Error); ok && gapiErr.Code == 404 {
viewTable := &gbigquery.Table{
TableReference: &gbigquery.TableReference{
DatasetId: self.datasetId,
ProjectId: self.projectId,
TableId: viewName,
},
View: &gbigquery.ViewDefinition{
Query: query,
},
}
if _, err = tablesService.Insert(self.projectId, self.datasetId, viewTable).Do(); err != nil {
if gapiErr, ok := err.(*googleapi.Error); ok && gapiErr.Code == 409 {
self.Infof("Unable to create %v, someone else already did it", viewName)
err = nil
return
} else {
err = errors.Errorf("Unable to create %#v with\n%v\n%v", viewName, utils.Prettify(viewTable), err)
return
}
}
}
}
return
}
/*
patchTable will add any fields in the provided type that happen to be missing from the originalTable
*/
func (self *BigQuery) patchTable(typ reflect.Type, tablesService *gbigquery.TablesService, originalTable *gbigquery.Table) (err error) {
table, err := self.buildTable(typ)
if err != nil {
return
}
unionTable := self.unionTables(table, originalTable)
if _, err = tablesService.Patch(self.projectId, self.datasetId, originalTable.TableReference.TableId, unionTable).Do(); err != nil {
err = errors.Errorf("Error trying to patch %#v with\n%v\n%v", typ.Name(), utils.Prettify(unionTable), err)
return
}
return
}
/*
createTable will create a table for the provided type inside BigQuery
*/
func (self *BigQuery) createTable(typ reflect.Type, tablesService *gbigquery.TablesService) (err error) {
table, err := self.buildTable(typ)
if err != nil {
return
}
if _, err = tablesService.Insert(self.projectId, self.datasetId, table).Do(); err != nil {
if gapiErr, ok := err.(*googleapi.Error); ok && gapiErr.Code == 409 {
self.Infof("Unable to create table for %v, someone else already did it", typ)
err = nil
return
}
err = errors.Errorf("Unable to create %#v with\n%v\n%v", typ.Name(), utils.Prettify(table), err)
return
}
return
}
/*
InsertTable data will assume that AssertTable for the type of the provided interface{} has been called beforehand, and
push this particular instance into BigQuery.
*/
func (self *BigQuery) InsertTableData(i interface{}) (err error) {
j := map[string]gbigquery.JsonValue{}
b, err := json.Marshal(i, "bigquery")
if err != nil {
return
}
if err = json.Unmarshal(b, &j); err != nil {
return
}
cropStrings(j)
if b, err = time.Now().MarshalJSON(); err != nil {
return
}
s := ""
if err = json.Unmarshal(b, &s); err != nil {
return
}
j["_inserted_at"] = s
request := &gbigquery.TableDataInsertAllRequest{
Rows: []*gbigquery.TableDataInsertAllRequestRows{
&gbigquery.TableDataInsertAllRequestRows{
Json: j,
},
},
}
typ := reflect.TypeOf(i)
for typ.Kind() == reflect.Ptr {
typ = typ.Elem()
}
for i := 0; i < typ.NumField(); i++ {
if typ.Field(i).Tag.Get("bigquery") == "-" {
name := typ.Field(i).Name
if jsonTag := typ.Field(i).Tag.Get("json"); jsonTag != "" {
if splitTag := strings.Split(jsonTag, ","); splitTag[0] != "" {
name = splitTag[0]
}
}
delete(j, name)
}
}
tabledataService := gbigquery.NewTabledataService(self.GetService())
tableDataList, err := tabledataService.InsertAll(self.GetProjectId(), self.GetDatasetId(), typ.Name(), request).Do()
if err != nil {
return
}
// Build insert errors error message
if len(tableDataList.InsertErrors) != 0 {
prettyJ := utils.Prettify(j)
errorStrings := []string{}
for _, errors := range tableDataList.InsertErrors {
for _, errorProto := range errors.Errors {
errorStrings = append(errorStrings, fmt.Sprintf("\nReason:%v,\nMessage:%v,\nLocation:%v", errorProto.Reason, errorProto.Message, errorProto.Location))
}
}
errorStrings = append(errorStrings, fmt.Sprintf("BigQuery: Error inserting json %v into table %v:", prettyJ, typ.Name()))
err = errors.Errorf(strings.Join(errorStrings, "\n"))
}
return
}
func Search(c ElasticSearchContext, query *SearchRequest, index, typ string) (result *SearchResponse, err error) {
if query.Size == 0 {
query.Size = 10
}
index = processIndexName(index)
url := c.GetElasticService()
if index == "" {
url += "/_all"
} else {
url += "/" + index
}
if typ != "" {
url += "/" + typ
}
url += "/_search"
b, err := json.Marshal(query)
if err != nil {
return
}
request, err := http.NewRequest("POST", url, bytes.NewBuffer(b))
if err != nil {
return
}
if c.GetElasticUsername() != "" {
request.SetBasicAuth(c.GetElasticUsername(), c.GetElasticPassword())
}
response, err := c.Client().Do(request)
if err != nil {
return
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
err = errors.Errorf("Bad status trying to search in elasticsearch %v: %v", url, response.Status)
return
}
result = &SearchResponse{}
bodyBytes, err := ioutil.ReadAll(response.Body)
if err != nil {
return
}
if err = json.Unmarshal(bodyBytes, &result); err != nil {
var secondTry interface{}
if err = json.Unmarshal(bodyBytes, &secondTry); err != nil {
return
}
err = errors.Errorf("Unable to marshal %v into %#v", utils.Prettify(secondTry), result)
return
}
c.Debugf("Elasticsearch took %v, url:%s", result.Took, url)
result.Page = 1 + (query.From / query.Size)
result.PerPage = query.Size
return
}