Esempio n. 1
0
/*
Create view of a table defined by a query.
*/
func (self *BigQuery) AssertView(viewName string, query string) (err error) {
	tablesService := gbigquery.NewTablesService(self.service)
	_, err = tablesService.Get(self.projectId, self.datasetId, viewName).Do()
	if err != nil {
		if gapiErr, ok := err.(*googleapi.Error); ok && gapiErr.Code == 404 {
			viewTable := &gbigquery.Table{
				TableReference: &gbigquery.TableReference{
					DatasetId: self.datasetId,
					ProjectId: self.projectId,
					TableId:   viewName,
				},
				View: &gbigquery.ViewDefinition{
					Query: query,
				},
			}
			if _, err = tablesService.Insert(self.projectId, self.datasetId, viewTable).Do(); err != nil {
				if gapiErr, ok := err.(*googleapi.Error); ok && gapiErr.Code == 409 {
					self.Infof("Unable to create %v, someone else already did it", viewName)
					err = nil
					return
				} else {
					err = errors.Errorf("Unable to create %#v with\n%v\n%v", viewName, utils.Prettify(viewTable), err)
					return
				}
			}
		}
	}
	return
}
Esempio n. 2
0
/*
patchTable will add any fields in the provided type that happen to be missing from the originalTable
*/
func (self *BigQuery) patchTable(typ reflect.Type, tablesService *gbigquery.TablesService, originalTable *gbigquery.Table) (err error) {

	table, err := self.buildTable(typ)
	if err != nil {
		return
	}

	unionTable := self.unionTables(table, originalTable)
	if _, err = tablesService.Patch(self.projectId, self.datasetId, originalTable.TableReference.TableId, unionTable).Do(); err != nil {
		err = errors.Errorf("Error trying to patch %#v with\n%v\n%v", typ.Name(), utils.Prettify(unionTable), err)
		return
	}
	return
}
Esempio n. 3
0
/*
createTable will create a table for the provided type inside BigQuery
*/
func (self *BigQuery) createTable(typ reflect.Type, tablesService *gbigquery.TablesService) (err error) {
	table, err := self.buildTable(typ)
	if err != nil {
		return
	}
	if _, err = tablesService.Insert(self.projectId, self.datasetId, table).Do(); err != nil {
		if gapiErr, ok := err.(*googleapi.Error); ok && gapiErr.Code == 409 {
			self.Infof("Unable to create table for %v, someone else already did it", typ)
			err = nil
			return
		}
		err = errors.Errorf("Unable to create %#v with\n%v\n%v", typ.Name(), utils.Prettify(table), err)
		return
	}
	return
}
Esempio n. 4
0
/*
InsertTable data will assume that AssertTable for the type of the provided interface{} has been called beforehand, and
push this particular instance into BigQuery.
*/
func (self *BigQuery) InsertTableData(i interface{}) (err error) {
	j := map[string]gbigquery.JsonValue{}

	b, err := json.Marshal(i, "bigquery")
	if err != nil {
		return
	}
	if err = json.Unmarshal(b, &j); err != nil {
		return
	}
	cropStrings(j)
	if b, err = time.Now().MarshalJSON(); err != nil {
		return
	}
	s := ""
	if err = json.Unmarshal(b, &s); err != nil {
		return
	}
	j["_inserted_at"] = s

	request := &gbigquery.TableDataInsertAllRequest{
		Rows: []*gbigquery.TableDataInsertAllRequestRows{
			&gbigquery.TableDataInsertAllRequestRows{
				Json: j,
			},
		},
	}

	typ := reflect.TypeOf(i)
	for typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
	}

	for i := 0; i < typ.NumField(); i++ {
		if typ.Field(i).Tag.Get("bigquery") == "-" {
			name := typ.Field(i).Name
			if jsonTag := typ.Field(i).Tag.Get("json"); jsonTag != "" {
				if splitTag := strings.Split(jsonTag, ","); splitTag[0] != "" {
					name = splitTag[0]
				}
			}
			delete(j, name)
		}
	}

	tabledataService := gbigquery.NewTabledataService(self.GetService())
	tableDataList, err := tabledataService.InsertAll(self.GetProjectId(), self.GetDatasetId(), typ.Name(), request).Do()
	if err != nil {
		return
	}

	// Build insert errors error message
	if len(tableDataList.InsertErrors) != 0 {
		prettyJ := utils.Prettify(j)
		errorStrings := []string{}
		for _, errors := range tableDataList.InsertErrors {
			for _, errorProto := range errors.Errors {
				errorStrings = append(errorStrings, fmt.Sprintf("\nReason:%v,\nMessage:%v,\nLocation:%v", errorProto.Reason, errorProto.Message, errorProto.Location))
			}
		}
		errorStrings = append(errorStrings, fmt.Sprintf("BigQuery: Error inserting json %v into table %v:", prettyJ, typ.Name()))
		err = errors.Errorf(strings.Join(errorStrings, "\n"))
	}

	return
}
Esempio n. 5
0
func Search(c ElasticSearchContext, query *SearchRequest, index, typ string) (result *SearchResponse, err error) {
	if query.Size == 0 {
		query.Size = 10
	}
	index = processIndexName(index)

	url := c.GetElasticService()
	if index == "" {
		url += "/_all"
	} else {
		url += "/" + index
	}
	if typ != "" {
		url += "/" + typ
	}
	url += "/_search"

	b, err := json.Marshal(query)
	if err != nil {
		return
	}

	request, err := http.NewRequest("POST", url, bytes.NewBuffer(b))
	if err != nil {
		return
	}

	if c.GetElasticUsername() != "" {
		request.SetBasicAuth(c.GetElasticUsername(), c.GetElasticPassword())
	}

	response, err := c.Client().Do(request)
	if err != nil {
		return
	}
	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		err = errors.Errorf("Bad status trying to search in elasticsearch %v: %v", url, response.Status)
		return
	}

	result = &SearchResponse{}
	bodyBytes, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return
	}
	if err = json.Unmarshal(bodyBytes, &result); err != nil {
		var secondTry interface{}
		if err = json.Unmarshal(bodyBytes, &secondTry); err != nil {
			return
		}
		err = errors.Errorf("Unable to marshal %v into %#v", utils.Prettify(secondTry), result)
		return
	}

	c.Debugf("Elasticsearch took %v, url:%s", result.Took, url)
	result.Page = 1 + (query.From / query.Size)
	result.PerPage = query.Size
	return
}