Example #1
0
// BuildBigQueryTables is a utility to configure BigQuery table for logging.
func BuildBigQueryTables(req *wcg.Request) error {
	if !LogSinkConfig.IsBigQueryEnabled() {
		return fmt.Errorf("BigQuery is not enabled, please check bigquery_project and bigquery_dataset configuration.")
	}

	client, err := bq.NewHTTPClient(NewContext(req))
	if err != nil {
		return err
	}
	service, err := bigquery.New(client)
	if err != nil {
		return err
	}
	projectID := LogSinkConfig.BigqueryProject
	datasetID := LogSinkConfig.BigqueryDataset
	for tableID, tbl := range LogSinkConfig.schema {
		tbl.TableReference.ProjectId = projectID
		tbl.TableReference.DatasetId = datasetID

		tbl1, err := service.Tables.Get(projectID, datasetID, tableID).Do()
		if err == nil {
			req.Logger.Debugf("Table %q already exists on \"%s/%s\"", tableID, projectID, datasetID)
			if len(tbl.Schema.Fields) < len(tbl1.Schema.Fields) {
				req.Logger.Fatalf("You cannot remove fields from table %q!!", tableID)
				continue
			}
			if len(tbl.Schema.Fields) == len(tbl1.Schema.Fields) {
				req.Logger.Infof("No filed changes on %q", tableID)
				continue
			}
			req.Logger.Infof("Trying to patch the exising table %q in %s/%s", tableID, LogSinkConfig.BigqueryProject, LogSinkConfig.BigqueryDataset)
			upd := service.Tables.Patch(LogSinkConfig.BigqueryProject, LogSinkConfig.BigqueryDataset, tableID, tbl)
			_, err = upd.Do()
			if err != nil {
				req.Logger.Errorf("Could not patch %q in %s/%s: %v", tableID, LogSinkConfig.BigqueryProject, LogSinkConfig.BigqueryDataset, err)
				continue
			}
			req.Logger.Infof("Table %q has been successfully updated in %s/%s", tableID, LogSinkConfig.BigqueryProject, LogSinkConfig.BigqueryDataset)
		} else {
			req.Logger.Infof("Trying to create a new table %q in %s/%s", tableID, LogSinkConfig.BigqueryProject, LogSinkConfig.BigqueryDataset)
			ins := service.Tables.Insert(LogSinkConfig.BigqueryProject, LogSinkConfig.BigqueryDataset, tbl)
			_, err = ins.Do()
			if err != nil {
				req.Logger.Errorf("Could not create a new table %q in %s/%s: %v", tableID, LogSinkConfig.BigqueryProject, LogSinkConfig.BigqueryDataset, err)
				continue
			}
			req.Logger.Infof("Table %q has been successfully created in %s/%s", tableID, LogSinkConfig.BigqueryProject, LogSinkConfig.BigqueryDataset)
		}
	}
	return nil
}
Example #2
0
// NewLogSinkWithContext creates a new *LogSink for GAE
func NewLogSinkWithContext(format string, ctx context.Context) *LogSink {
	var _bq *bigquery.Service
	if LogSinkConfig.IsBigQueryEnabled() {
		client, err := bq.NewHTTPClient(ctx)
		if err == nil {
			_bq, _ = bigquery.New(client)
		}
	}
	return &LogSink{
		formatter: wcg.NewLogRecordFormatter(format),
		ctx:       ctx,
		bigquery:  _bq,
	}
}