Example #1
0
func main() {
	flag.Parse()

	flagsOk := true
	if flag.Lookup("project").Value.String() == "" {
		fmt.Fprintf(os.Stderr, "Flag --project is required\n")
		flagsOk = false
	}

	var sourceFlagCount int
	if flag.Lookup("dataset").Value.String() != "" {
		sourceFlagCount++
	}
	if flag.Lookup("jobid").Value.String() != "" {
		sourceFlagCount++
	}
	if sourceFlagCount != 1 {
		fmt.Fprintf(os.Stderr, "Exactly one of --dataset or --jobid must be set\n")
		flagsOk = false
	}

	if !flagsOk {
		os.Exit(1)
	}

	tableRE, err := regexp.Compile(*table)
	if err != nil {
		fmt.Fprintf(os.Stderr, "--table is not a valid regular expression: %q\n", *table)
		os.Exit(1)
	}

	httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope)
	if err != nil {
		log.Fatalf("Creating http client: %v", err)
	}

	client, err := bigquery.NewClient(httpClient, *project)
	if err != nil {
		log.Fatalf("Creating bigquery client: %v", err)
	}

	if *jobID != "" {
		printQueryResults(client, *jobID)
		return
	}
	ds := client.Dataset(*dataset)
	var tables []*bigquery.Table
	tables, err = ds.ListTables(context.Background())
	if err != nil {
		log.Fatalf("Listing tables: %v", err)
	}
	for _, t := range tables {
		if tableRE.MatchString(t.TableID) {
			printTable(client, t)
		}
	}
}
Example #2
0
func main() {
	flag.Parse()

	flagsOk := true
	for _, f := range []string{"project", "dataset", "src1", "src2", "dest"} {
		if flag.Lookup(f).Value.String() == "" {
			fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
			flagsOk = false
		}
	}
	if !flagsOk {
		os.Exit(1)
	}
	if *src1 == *src2 || *src1 == *dest || *src2 == *dest {
		log.Fatalf("Different values must be supplied for each of --src1, --src2 and --dest")
	}

	httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope)
	if err != nil {
		log.Fatalf("Creating http client: %v", err)
	}

	client, err := bigquery.NewClient(httpClient, *project)
	if err != nil {
		log.Fatalf("Creating bigquery client: %v", err)
	}

	s1 := &bigquery.Table{
		ProjectID: *project,
		DatasetID: *dataset,
		TableID:   *src1,
	}

	s2 := &bigquery.Table{
		ProjectID: *project,
		DatasetID: *dataset,
		TableID:   *src2,
	}

	d := &bigquery.Table{
		ProjectID: *project,
		DatasetID: *dataset,
		TableID:   *dest,
	}

	// Concatenate data.
	job, err := client.Copy(context.Background(), d, bigquery.Tables{s1, s2}, bigquery.WriteTruncate)

	if err != nil {
		log.Fatalf("Concatenating: %v", err)
	}

	fmt.Printf("Job for concatenation operation: %+v\n", job)
	fmt.Printf("Waiting for job to complete.\n")

	for range time.Tick(*pollint) {
		status, err := job.Status(context.Background())
		if err != nil {
			fmt.Printf("Failure determining status: %v", err)
			break
		}
		if !status.Done() {
			continue
		}
		if err := status.Err(); err == nil {
			fmt.Printf("Success\n")
		} else {
			fmt.Printf("Failure: %+v\n", err)
		}
		break
	}
}
Example #3
0
func main() {
	flag.Parse()

	flagsOk := true
	for _, f := range []string{"project", "dataset", "table", "bucket", "object"} {
		if flag.Lookup(f).Value.String() == "" {
			fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
			flagsOk = false
		}
	}
	if !flagsOk {
		os.Exit(1)
	}

	httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope)
	if err != nil {
		log.Fatalf("Creating http client: %v", err)
	}

	client, err := bigquery.NewClient(httpClient, *project)
	if err != nil {
		log.Fatalf("Creating bigquery client: %v", err)
	}

	table := &bigquery.Table{
		ProjectID: *project,
		DatasetID: *dataset,
		TableID:   *table,
	}

	gcs := client.NewGCSReference(fmt.Sprintf("gs://%s/%s", *bucket, *object))
	gcs.SkipLeadingRows = *skiprows

	// Load data from Google Cloud Storage into a BigQuery table.
	job, err := client.Copy(
		context.Background(), table, gcs,
		bigquery.MaxBadRecords(1),
		bigquery.AllowQuotedNewlines(),
		bigquery.WriteTruncate)

	if err != nil {
		log.Fatalf("Loading data: %v", err)
	}

	fmt.Printf("Job for data load operation: %+v\n", job)
	fmt.Printf("Waiting for job to complete.\n")

	for range time.Tick(*pollint) {
		status, err := job.Status(context.Background())
		if err != nil {
			fmt.Printf("Failure determining status: %v", err)
			break
		}
		if !status.Done() {
			continue
		}
		if err := status.Err(); err == nil {
			fmt.Printf("Success\n")
		} else {
			fmt.Printf("Failure: %+v\n", err)
		}
		break
	}
}
Example #4
0
func main() {
	flag.Parse()

	flagsOk := true
	for _, f := range []string{"project", "dataset", "q"} {
		if flag.Lookup(f).Value.String() == "" {
			fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
			flagsOk = false
		}
	}
	if !flagsOk {
		os.Exit(1)
	}

	httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope)
	if err != nil {
		log.Fatalf("Creating http client: %v", err)
	}

	client, err := bigquery.NewClient(httpClient, *project)
	if err != nil {
		log.Fatalf("Creating bigquery client: %v", err)
	}

	d := &bigquery.Table{}

	if *dest != "" {
		d.ProjectID = *project
		d.DatasetID = *dataset
		d.TableID = *dest
	}

	query := &bigquery.Query{
		Q:                *q,
		DefaultProjectID: *project,
		DefaultDatasetID: *dataset,
	}

	// Query data.
	job, err := client.Copy(context.Background(), d, query, bigquery.WriteTruncate)

	if err != nil {
		log.Fatalf("Querying: %v", err)
	}

	fmt.Printf("Submitted query. Job ID: %s\n", job.ID())
	if !*wait {
		return
	}

	fmt.Printf("Waiting for job to complete.\n")

	for range time.Tick(*pollint) {
		status, err := job.Status(context.Background())
		if err != nil {
			fmt.Printf("Failure determining status: %v", err)
			break
		}
		if !status.Done() {
			continue
		}
		if err := status.Err(); err == nil {
			fmt.Printf("Success\n")
		} else {
			fmt.Printf("Failure: %+v\n", err)
		}
		break
	}
}