// BuildBigQueryTables is a utility to configure BigQuery table for logging.
func BuildBigQueryTables(req *wcg.Request) error {
if !LogSinkConfig.IsBigQueryEnabled() {
return fmt.Errorf("BigQuery is not enabled, please check bigquery_project and bigquery_dataset configuration.")
}
client, err := bq.NewHTTPClient(NewContext(req))
if err != nil {
return err
}
service, err := bigquery.New(client)
if err != nil {
return err
}
projectID := LogSinkConfig.BigqueryProject
datasetID := LogSinkConfig.BigqueryDataset
for tableID, tbl := range LogSinkConfig.schema {
tbl.TableReference.ProjectId = projectID
tbl.TableReference.DatasetId = datasetID
tbl1, err := service.Tables.Get(projectID, datasetID, tableID).Do()
if err == nil {
req.Logger.Debugf("Table %q already exists on \"%s/%s\"", tableID, projectID, datasetID)
if len(tbl.Schema.Fields) < len(tbl1.Schema.Fields) {
req.Logger.Fatalf("You cannot remove fields from table %q!!", tableID)
continue
}
if len(tbl.Schema.Fields) == len(tbl1.Schema.Fields) {
req.Logger.Infof("No filed changes on %q", tableID)
continue
}
req.Logger.Infof("Trying to patch the exising table %q in %s/%s", tableID, LogSinkConfig.BigqueryProject, LogSinkConfig.BigqueryDataset)
upd := service.Tables.Patch(LogSinkConfig.BigqueryProject, LogSinkConfig.BigqueryDataset, tableID, tbl)
_, err = upd.Do()
if err != nil {
req.Logger.Errorf("Could not patch %q in %s/%s: %v", tableID, LogSinkConfig.BigqueryProject, LogSinkConfig.BigqueryDataset, err)
continue
}
req.Logger.Infof("Table %q has been successfully updated in %s/%s", tableID, LogSinkConfig.BigqueryProject, LogSinkConfig.BigqueryDataset)
} else {
req.Logger.Infof("Trying to create a new table %q in %s/%s", tableID, LogSinkConfig.BigqueryProject, LogSinkConfig.BigqueryDataset)
ins := service.Tables.Insert(LogSinkConfig.BigqueryProject, LogSinkConfig.BigqueryDataset, tbl)
_, err = ins.Do()
if err != nil {
req.Logger.Errorf("Could not create a new table %q in %s/%s: %v", tableID, LogSinkConfig.BigqueryProject, LogSinkConfig.BigqueryDataset, err)
continue
}
req.Logger.Infof("Table %q has been successfully created in %s/%s", tableID, LogSinkConfig.BigqueryProject, LogSinkConfig.BigqueryDataset)
}
}
return nil
}
// NewLogSinkWithContext creates a new *LogSink for GAE
func NewLogSinkWithContext(format string, ctx context.Context) *LogSink {
var _bq *bigquery.Service
if LogSinkConfig.IsBigQueryEnabled() {
client, err := bq.NewHTTPClient(ctx)
if err == nil {
_bq, _ = bigquery.New(client)
}
}
return &LogSink{
formatter: wcg.NewLogRecordFormatter(format),
ctx: ctx,
bigquery: _bq,
}
}