func main() {
flag.Parse()
flagsOk := true
for _, f := range []string{"project", "dataset", "table", "bucket", "object"} {
if flag.Lookup(f).Value.String() == "" {
fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
flagsOk = false
}
}
if !flagsOk {
os.Exit(1)
}
ctx := context.Background()
client, err := bigquery.NewClient(ctx, *project)
if err != nil {
log.Fatalf("Creating bigquery client: %v", err)
}
table := &bigquery.Table{
ProjectID: *project,
DatasetID: *dataset,
TableID: *table,
}
gcs := client.NewGCSReference(fmt.Sprintf("gs://%s/%s", *bucket, *object))
gcs.SkipLeadingRows = *skiprows
// Load data from Google Cloud Storage into a BigQuery table.
job, err := client.Copy(
ctx, table, gcs,
bigquery.MaxBadRecords(1),
bigquery.AllowQuotedNewlines(),
bigquery.WriteTruncate)
if err != nil {
log.Fatalf("Loading data: %v", err)
}
fmt.Printf("Job for data load operation: %+v\n", job)
fmt.Printf("Waiting for job to complete.\n")
for range time.Tick(*pollint) {
status, err := job.Status(ctx)
if err != nil {
fmt.Printf("Failure determining status: %v", err)
break
}
if !status.Done() {
continue
}
if err := status.Err(); err == nil {
fmt.Printf("Success\n")
} else {
fmt.Printf("Failure: %+v\n", err)
}
break
}
}
func main() {
flag.Parse()
flagsOk := true
if flag.Lookup("project").Value.String() == "" {
fmt.Fprintf(os.Stderr, "Flag --project is required\n")
flagsOk = false
}
var sourceFlagCount int
if flag.Lookup("dataset").Value.String() != "" {
sourceFlagCount++
}
if flag.Lookup("jobid").Value.String() != "" {
sourceFlagCount++
}
if sourceFlagCount != 1 {
fmt.Fprintf(os.Stderr, "Exactly one of --dataset or --jobid must be set\n")
flagsOk = false
}
if !flagsOk {
os.Exit(1)
}
tableRE, err := regexp.Compile(*table)
if err != nil {
fmt.Fprintf(os.Stderr, "--table is not a valid regular expression: %q\n", *table)
os.Exit(1)
}
httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope)
if err != nil {
log.Fatalf("Creating http client: %v", err)
}
client, err := bigquery.NewClient(httpClient, *project)
if err != nil {
log.Fatalf("Creating bigquery client: %v", err)
}
if *jobID != "" {
printQueryResults(client, *jobID)
return
}
ds := client.Dataset(*dataset)
var tables []*bigquery.Table
tables, err = ds.ListTables(context.Background())
if err != nil {
log.Fatalf("Listing tables: %v", err)
}
for _, t := range tables {
if tableRE.MatchString(t.TableID) {
printTable(client, t)
}
}
}
func main() {
flag.Parse()
flagsOk := true
for _, f := range []string{"project", "dataset", "table"} {
if flag.Lookup(f).Value.String() == "" {
fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
flagsOk = false
}
}
if !flagsOk {
os.Exit(1)
}
httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope)
if err != nil {
log.Fatalf("Creating http client: %v", err)
}
client, err := bigquery.NewClient(httpClient, *project)
if err != nil {
log.Fatalf("Creating bigquery client: %v", err)
}
it, err := client.Read(&bigquery.Table{
ProjectID: *project,
DatasetID: *dataset,
TableID: *table,
})
if err != nil {
log.Fatalf("Reading: %v", err)
}
// one-space padding.
tw := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
for it.Next(context.Background()) {
var vals bigquery.ValueList
if err := it.Get(&vals); err != nil {
fmt.Printf("err calling get: %v\n", err)
} else {
sep := ""
for _, v := range vals {
fmt.Fprintf(tw, "%s%v", sep, v)
sep = "\t"
}
fmt.Fprintf(tw, "\n")
}
}
tw.Flush()
if err := it.Err(); err != nil {
fmt.Printf("err reading: %v\n")
}
}
func main() {
flag.Parse()
flagsOk := true
for _, f := range []string{"project", "dataset"} {
if flag.Lookup(f).Value.String() == "" {
fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
flagsOk = false
}
}
if !flagsOk {
os.Exit(1)
}
tableRE, err := regexp.Compile(*table)
if err != nil {
fmt.Fprintf(os.Stderr, "--table is not a valid regular expression: %q\n", *table)
os.Exit(1)
}
httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope)
if err != nil {
log.Fatalf("Creating http client: %v", err)
}
client, err := bigquery.NewClient(httpClient, *project)
if err != nil {
log.Fatalf("Creating bigquery client: %v", err)
}
ds := client.Dataset(*dataset)
var tables []*bigquery.Table
tables, err = ds.ListTables(context.Background())
if err != nil {
log.Fatalf("Listing tables: %v", err)
}
for _, t := range tables {
if tableRE.MatchString(t.TableID) {
printTable(client, t)
}
}
}
// https://sourcegraph.com/github.com/GoogleCloudPlatform/gcloud-golang/-/blob/examples/bigquery/load/main.go
func submitLoadJob(r *http.Request, gcsfolder, gcsfile string) error {
ctx := appengine.NewContext(r)
client, err := bigquery.NewClient(ctx, bigqueryProject)
if err != nil {
return fmt.Errorf("Creating bigquery client: %v", err)
}
gcsSrc := client.NewGCSReference(fmt.Sprintf("gs://%s/%s", gcsfolder, gcsfile))
gcsSrc.SourceFormat = bigquery.JSON
tableDest := &bigquery.Table{
ProjectID: bigqueryProject,
DatasetID: bigqueryDataset,
TableID: bigqueryTableName,
}
job, err := client.Copy(ctx, tableDest, gcsSrc, bigquery.WriteAppend)
if err != nil {
return fmt.Errorf("Submission of load job: %v", err)
}
time.Sleep(5 * time.Second)
if status, err := job.Status(ctx); err != nil {
return fmt.Errorf("Failure determining status: %v", err)
} else if err := status.Err(); err != nil {
detailedErrStr := ""
for i, innerErr := range status.Errors {
detailedErrStr += fmt.Sprintf(" [%2d] %v\n", i, innerErr)
}
log.Errorf(ctx, "BiqQuery LoadJob error: %v\n--\n%s", err, detailedErrStr)
return fmt.Errorf("Job error: %v\n--\n%s", err, detailedErrStr)
} else {
log.Infof(ctx, "BiqQuery LoadJob status: done=%v, state=%s, %s",
status.Done(), status.State, status)
}
return nil
}
func main() {
flag.Parse()
flagsOk := true
for _, f := range []string{"project", "dataset", "q"} {
if flag.Lookup(f).Value.String() == "" {
fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
flagsOk = false
}
}
if !flagsOk {
os.Exit(1)
}
httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope)
if err != nil {
log.Fatalf("Creating http client: %v", err)
}
client, err := bigquery.NewClient(httpClient, *project)
if err != nil {
log.Fatalf("Creating bigquery client: %v", err)
}
d := &bigquery.Table{
WriteDisposition: bigquery.WriteTruncate,
}
if *dest != "" {
d.ProjectID = *project
d.DatasetID = *dataset
d.TableID = *dest
}
query := &bigquery.Query{
Q: *q,
DefaultProjectID: *project,
DefaultDatasetID: *dataset,
}
// Query data.
job, err := client.Copy(context.Background(), d, query)
if err != nil {
log.Fatalf("Querying: %v", err)
}
fmt.Printf("Submitted query. Job ID: %s\n", job.ID())
if !*wait {
return
}
fmt.Printf("Waiting for job to complete.\n")
for range time.Tick(*pollint) {
status, err := job.Status(context.Background())
if err != nil {
fmt.Printf("Failure determining status: %v", err)
break
}
if !status.Done() {
continue
}
if err := status.Err(); err == nil {
fmt.Printf("Success\n")
} else {
fmt.Printf("Failure: %+v\n", err)
}
break
}
}
func main() {
flag.Parse()
flagsOk := true
for _, f := range []string{"project", "dataset", "src1", "src2", "dest"} {
if flag.Lookup(f).Value.String() == "" {
fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
flagsOk = false
}
}
if !flagsOk {
os.Exit(1)
}
if *src1 == *src2 || *src1 == *dest || *src2 == *dest {
log.Fatalf("Different values must be supplied for each of --src1, --src2 and --dest")
}
httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope)
if err != nil {
log.Fatalf("Creating http client: %v", err)
}
client, err := bigquery.NewClient(httpClient, *project)
if err != nil {
log.Fatalf("Creating bigquery client: %v", err)
}
s1 := &bigquery.Table{
ProjectID: *project,
DatasetID: *dataset,
TableID: *src1,
}
s2 := &bigquery.Table{
ProjectID: *project,
DatasetID: *dataset,
TableID: *src2,
}
d := &bigquery.Table{
ProjectID: *project,
DatasetID: *dataset,
TableID: *dest,
}
// Concatenate data.
job, err := client.Copy(context.Background(), d, bigquery.Tables{s1, s2}, bigquery.WriteTruncate)
if err != nil {
log.Fatalf("Concatenating: %v", err)
}
fmt.Printf("Job for concatenation operation: %+v\n", job)
fmt.Printf("Waiting for job to complete.\n")
for range time.Tick(*pollint) {
status, err := job.Status(context.Background())
if err != nil {
fmt.Printf("Failure determining status: %v", err)
break
}
if !status.Done() {
continue
}
if err := status.Err(); err == nil {
fmt.Printf("Success\n")
} else {
fmt.Printf("Failure: %+v\n", err)
}
break
}
}