func main() {

	var bigtableClientConnData = gbigtable.ClientConnectionData{
		Project:         "bigdatagarage",
		Zone:            "us-central1-c",
		Cluster:         "workshopanalytics",
		KeyJsonFilePath: KeyJsonFilePath,
	}

	// https://godoc.org/google.golang.org/cloud/bigtable#RowRange
	// rr := bigtable.PrefixRange("my")
	rr := bigtable.InfiniteRange("")
	// rr := bigtable.NewRange("begin", "end")
	// rr := bigtable.SingleRow("mykey01")

	printFilteredRowsFromTable(bigtableClientConnData, "test-tmp", rr, nil)

	// https://godoc.org/google.golang.org/cloud/bigtable#Filter
	var opts []bigtable.ReadOption
	// opts = append(opts, bigtable.RowFilter(bigtable.FamilyFilter(".*fam01.*")));
	// opts = append(opts, bigtable.RowFilter(bigtable.ColumnFilter(".*col.*")));
	opts = append(opts, bigtable.RowFilter(bigtable.ColumnFilter(".*col02.*")))
	// opts = append(opts, bigtable.RowFilter(bigtable.ChainFilters(bigtable.ColumnFilter(".*col.*"), bigtable.ValueFilter(".*val.*"))));

	printFilteredRowsFromTable(bigtableClientConnData, "test-tmp", (bigtable.RowRange{}), opts)
}
Beispiel #2
0
// rebuildTable deletes the table if it exists, then creates the table, with the index column family.
func rebuildTable() error {
	ctx, _ := context.WithTimeout(context.Background(), 5*time.Minute)
	adminClient.DeleteTable(ctx, *tableName)
	if err := adminClient.CreateTable(ctx, *tableName); err != nil {
		return fmt.Errorf("CreateTable: %v", err)
	}
	time.Sleep(20 * time.Second)
	if err := adminClient.CreateColumnFamily(ctx, *tableName, indexColumnFamily); err != nil {
		return fmt.Errorf("CreateColumnFamily: %v", err)
	}
	if err := adminClient.CreateColumnFamily(ctx, *tableName, contentColumnFamily); err != nil {
		return fmt.Errorf("CreateColumnFamily: %v", err)
	}

	// Open the prototype table.  It contains a number of documents to get started with.
	prototypeTable := client.Open(prototypeTableName)

	var (
		writeErr error          // Set if any write fails.
		mu       sync.Mutex     // Protects writeErr
		wg       sync.WaitGroup // Used to wait for all writes to finish.
	)
	copyRowToTable := func(row bigtable.Row) bool {
		mu.Lock()
		failed := writeErr != nil
		mu.Unlock()
		if failed {
			return false
		}
		mut := bigtable.NewMutation()
		for family, items := range row {
			for _, item := range items {
				// Get the column name, excluding the column family name and ':' character.
				columnWithoutFamily := item.Column[len(family)+1:]
				mut.Set(family, columnWithoutFamily, bigtable.Now(), item.Value)
			}
		}
		wg.Add(1)
		go func() {
			// TODO: should use a semaphore to limit the number of concurrent writes.
			if err := table.Apply(ctx, row.Key(), mut); err != nil {
				mu.Lock()
				writeErr = err
				mu.Unlock()
			}
			wg.Done()
		}()
		return true
	}

	// Create a filter that only accepts the column families we're interested in.
	filter := bigtable.FamilyFilter(indexColumnFamily + "|" + contentColumnFamily)
	// Read every row from prototypeTable, and call copyRowToTable to copy it to our table.
	err := prototypeTable.ReadRows(ctx, bigtable.InfiniteRange(""), copyRowToTable, bigtable.RowFilter(filter))
	wg.Wait()
	if err != nil {
		return err
	}
	return writeErr
}
Beispiel #3
0
func doRead(ctx context.Context, args ...string) {
	if len(args) < 1 {
		log.Fatalf("usage: cbt read <table> [args ...]")
	}
	tbl := getClient().Open(args[0])

	parsed := make(map[string]string)
	for _, arg := range args[1:] {
		i := strings.Index(arg, "=")
		if i < 0 {
			log.Fatalf("Bad arg %q", arg)
		}
		key, val := arg[:i], arg[i+1:]
		switch key {
		default:
			log.Fatalf("Unknown arg key %q", key)
		case "limit":
			// Be nicer; we used to support this, but renamed it to "end".
			log.Fatalf("Unknown arg key %q; did you mean %q?", key, "end")
		case "start", "end", "prefix", "count":
			parsed[key] = val
		}
	}
	if (parsed["start"] != "" || parsed["end"] != "") && parsed["prefix"] != "" {
		log.Fatal(`"start"/"end" may not be mixed with "prefix"`)
	}

	var rr bigtable.RowRange
	if start, end := parsed["start"], parsed["end"]; end != "" {
		rr = bigtable.NewRange(start, end)
	} else if start != "" {
		rr = bigtable.InfiniteRange(start)
	}
	if prefix := parsed["prefix"]; prefix != "" {
		rr = bigtable.PrefixRange(prefix)
	}

	var opts []bigtable.ReadOption
	if count := parsed["count"]; count != "" {
		n, err := strconv.ParseInt(count, 0, 64)
		if err != nil {
			log.Fatalf("Bad count %q: %v", count, err)
		}
		opts = append(opts, bigtable.LimitRows(n))
	}

	// TODO(dsymonds): Support filters.
	err := tbl.ReadRows(ctx, rr, func(r bigtable.Row) bool {
		printRow(r)
		return true
	}, opts...)
	if err != nil {
		log.Fatalf("Reading rows: %v", err)
	}
}
Beispiel #4
0
// copyTable copies data from one table to another.
func copyTable(src, dst string, client *bigtable.Client, adminClient *bigtable.AdminClient) error {
	if src == "" || src == dst {
		return nil
	}
	ctx, _ := context.WithTimeout(context.Background(), time.Minute)

	// Open the source and destination tables.
	srcTable := client.Open(src)
	dstTable := client.Open(dst)

	var (
		writeErr error          // Set if any write fails.
		mu       sync.Mutex     // Protects writeErr
		wg       sync.WaitGroup // Used to wait for all writes to finish.
	)
	copyRowToTable := func(row bigtable.Row) bool {
		mu.Lock()
		failed := writeErr != nil
		mu.Unlock()
		if failed {
			return false
		}
		mut := bigtable.NewMutation()
		for family, items := range row {
			for _, item := range items {
				// Get the column name, excluding the column family name and ':' character.
				columnWithoutFamily := item.Column[len(family)+1:]
				mut.Set(family, columnWithoutFamily, bigtable.Now(), item.Value)
			}
		}
		wg.Add(1)
		go func() {
			// TODO: should use a semaphore to limit the number of concurrent writes.
			if err := dstTable.Apply(ctx, row.Key(), mut); err != nil {
				mu.Lock()
				writeErr = err
				mu.Unlock()
			}
			wg.Done()
		}()
		return true
	}

	// Create a filter that only accepts the column families we're interested in.
	filter := bigtable.FamilyFilter(indexColumnFamily + "|" + contentColumnFamily)
	// Read every row from srcTable, and call copyRowToTable to copy it to our table.
	err := srcTable.ReadRows(ctx, bigtable.InfiniteRange(""), copyRowToTable, bigtable.RowFilter(filter))
	wg.Wait()
	if err != nil {
		return err
	}
	return writeErr
}
Beispiel #5
0
func main() {
	flag.Parse()
	ctx := context.Background()
	os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "grpc-go16-dev-608d72c9bce0.json")
	const tableName = "mytable"
	if *create {
		ac, err := bigtable.NewAdminClient(ctx, "grpc-go16-dev", "us-central1-c", "dollar-95-per-hour")
		if err != nil {
			log.Fatalf("AdminClient: %v", err)
		}
		if err := ac.CreateTable(ctx, tableName); err != nil {
			log.Printf("Create: %v", err)
		}
		if err := ac.CreateColumnFamily(ctx, tableName, "fam"); err != nil {
			log.Printf("CreateColumnFamily: %v", err)
		}
	}

	bc, err := bigtable.NewClient(ctx, "grpc-go16-dev", "us-central1-c", "dollar-95-per-hour")
	if err != nil {
		log.Fatalf("NewClient: %v", err)
	}
	t := bc.Open(tableName)
	m := bigtable.NewMutation()
	m.Set("fam", "col", bigtable.Now(), []byte("hello"))

	if *insert {
		log.Printf("Apply...")
		err = t.Apply(ctx, fmt.Sprintf("t:%v", time.Now().Unix()), m)
		if err != nil {
			log.Fatalf("Apply: %v", err)
		}
	}

	log.Printf("ReadRows...")

	t.ReadRows(ctx, bigtable.InfiniteRange(""), func(row bigtable.Row) bool {
		log.Printf("Row: %v", row.Key())
		for fam, ris := range row {
			log.Printf("  col fam %q", fam)
			for _, ri := range ris {
				log.Printf("    row %q, col %q, ts %v, val %q", ri.Row, ri.Column, ri.Timestamp, ri.Value)
			}
		}
		return true
	})
	log.Printf("done.")
}
Beispiel #6
0
func doCount(ctx context.Context, args ...string) {
	if len(args) != 1 {
		log.Fatal("usage: cbt count <table>")
	}
	tbl := getClient().Open(args[0])

	n := 0
	err := tbl.ReadRows(ctx, bigtable.InfiniteRange(""), func(_ bigtable.Row) bool {
		n++
		return true
	}, bigtable.RowFilter(bigtable.StripValueFilter()))
	if err != nil {
		log.Fatalf("Reading rows: %v", err)
	}
	fmt.Println(n)
}
Beispiel #7
0
func doRead(ctx context.Context, args ...string) {
	if len(args) < 1 {
		log.Fatalf("usage: cbt read <table> [args ...]")
	}
	tbl := getClient().Open(args[0])

	parsed := make(map[string]string)
	for _, arg := range args[1:] {
		i := strings.Index(arg, "=")
		if i < 0 {
			log.Fatalf("Bad arg %q", arg)
		}
		key, val := arg[:i], arg[i+1:]
		switch key {
		default:
			log.Fatalf("Unknown arg key %q", key)
		case "start", "limit", "prefix":
			parsed[key] = val
		}
	}
	if (parsed["start"] != "" || parsed["limit"] != "") && parsed["prefix"] != "" {
		log.Fatal(`"start"/"limit" may not be mixed with "prefix"`)
	}

	var rr bigtable.RowRange
	if start, limit := parsed["start"], parsed["limit"]; limit != "" {
		rr = bigtable.NewRange(start, limit)
	} else if start != "" {
		rr = bigtable.InfiniteRange(start)
	}
	if prefix := parsed["prefix"]; prefix != "" {
		rr = bigtable.PrefixRange(prefix)
	}

	// TODO(dsymonds): Support filters.
	err := tbl.ReadRows(ctx, rr, func(r bigtable.Row) bool {
		printRow(r)
		return true
	})
	if err != nil {
		log.Fatalf("Reading rows: %v", err)
	}
}
func ReadAllRows(ctx context.Context, table *bigtable.Table, processDataTableRow ProcessDataTableRow) {

	ReadRows(ctx, table, bigtable.InfiniteRange(""), processDataTableRow)
}