func main() {
var (
project = flag.String("project", "", "The name of the project.")
instance = flag.String("instance", "", "The name of the Cloud Bigtable instance.")
tableName = flag.String("table", "docindex", "The name of the table containing the documents and index.")
port = flag.Int("port", 8080, "TCP port for server.")
)
flag.Parse()
// Make an admin client.
adminClient, err := bigtable.NewAdminClient(context.Background(), *project, *instance)
if err != nil {
log.Fatal("Bigtable NewAdminClient:", err)
}
// Make a regular client.
client, err := bigtable.NewClient(context.Background(), *project, *instance)
if err != nil {
log.Fatal("Bigtable NewClient:", err)
}
// Open the table.
table := client.Open(*tableName)
// Set up HTML handlers, and start the web server.
http.HandleFunc("/search", func(w http.ResponseWriter, r *http.Request) { handleSearch(w, r, table) })
http.HandleFunc("/content", func(w http.ResponseWriter, r *http.Request) { handleContent(w, r, table) })
http.HandleFunc("/add", func(w http.ResponseWriter, r *http.Request) { handleAddDoc(w, r, table) })
http.HandleFunc("/reset", func(w http.ResponseWriter, r *http.Request) { handleReset(w, r, *tableName, adminClient) })
http.HandleFunc("/copy", func(w http.ResponseWriter, r *http.Request) { handleCopy(w, r, *tableName, client, adminClient) })
http.HandleFunc("/", handleMain)
log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*port), nil))
}
func getClient() *bigtable.Client {
if client == nil {
var err error
client, err = bigtable.NewClient(context.Background(), config.Project, config.Instance)
if err != nil {
log.Fatalf("Making bigtable.Client: %v", err)
}
}
return client
}
func ExampleNewServer() {
srv, err := bttest.NewServer("127.0.0.1:0")
if err != nil {
log.Fatalln(err)
}
ctx := context.Background()
conn, err := grpc.Dial(srv.Addr, grpc.WithInsecure())
if err != nil {
log.Fatalln(err)
}
proj, instance := "proj", "instance"
adminClient, err := bigtable.NewAdminClient(ctx, proj, instance, option.WithGRPCConn(conn))
if err != nil {
log.Fatalln(err)
}
if err = adminClient.CreateTable(ctx, "example"); err != nil {
log.Fatalln(err)
}
if err = adminClient.CreateColumnFamily(ctx, "example", "links"); err != nil {
log.Fatalln(err)
}
client, err := bigtable.NewClient(ctx, proj, instance, option.WithGRPCConn(conn))
if err != nil {
log.Fatalln(err)
}
tbl := client.Open("example")
mut := bigtable.NewMutation()
mut.Set("links", "golang.org", bigtable.Now(), []byte("Gophers!"))
if err = tbl.Apply(ctx, "com.google.cloud", mut); err != nil {
log.Fatalln(err)
}
if row, err := tbl.ReadRow(ctx, "com.google.cloud"); err != nil {
log.Fatalln(err)
} else {
for _, column := range row["links"] {
fmt.Println(column.Column)
fmt.Println(string(column.Value))
}
}
// Output:
// links:golang.org
// Gophers!
}
func NewClient(bt *BigTable) (client *api.Client, err error) {
if bt.testing {
testSrv, err := bttest.NewServer() //TODO close the testSrv if neccesary
if err != nil {
return nil, fmt.Errorf("Unable to create bigTable local test server. %v", err)
}
testConn, err := grpc.Dial(testSrv.Addr, grpc.WithInsecure())
if err != nil {
return nil, fmt.Errorf("Unable to create bigTable local test server. %v", err)
}
//Connects to a local bigtable.
//It with no security; The project, zone, cluster values are ignored.
client, err = api.NewClient(bt.ctx, bt.project, bt.zone, bt.cluster, option.WithGRPCConn(testConn))
} else {
//Uses Application Default Credentials to authenticate into Google's Cloud.
client, err = api.NewClient(bt.ctx, bt.project, bt.zone, bt.cluster)
}
if err != nil {
return nil, fmt.Errorf("Unable to create a table client. %v", err)
}
return
}
func getClient() *bigtable.Client {
if client == nil {
var opts []option.ClientOption
if ep := config.DataEndpoint; ep != "" {
opts = append(opts, option.WithEndpoint(ep))
}
var err error
client, err = bigtable.NewClient(context.Background(), config.Project, config.Instance, opts...)
if err != nil {
log.Fatalf("Making bigtable.Client: %v", err)
}
}
return client
}
func main() {
ctx := context.Background()
// Set up admin client, tables, and column families.
// NewAdminClient uses Application Default Credentials to authenticate.
adminClient, err := bigtable.NewAdminClient(ctx, project, instance)
if err != nil {
log.Fatalf("Unable to create a table admin client. %v", err)
}
tables, err := adminClient.Tables(ctx)
if err != nil {
log.Fatalf("Unable to fetch table list. %v", err)
}
if !sliceContains(tables, tableName) {
if err := adminClient.CreateTable(ctx, tableName); err != nil {
log.Fatalf("Unable to create table: %v. %v", tableName, err)
}
}
tblInfo, err := adminClient.TableInfo(ctx, tableName)
if err != nil {
log.Fatalf("Unable to read info for table: %v. %v", tableName, err)
}
if !sliceContains(tblInfo.Families, familyName) {
if err := adminClient.CreateColumnFamily(ctx, tableName, familyName); err != nil {
log.Fatalf("Unable to create column family: %v. %v", familyName, err)
}
}
adminClient.Close()
// Set up Bigtable data operations client.
// NewClient uses Application Default Credentials to authenticate.
client, err = bigtable.NewClient(ctx, project, instance)
if err != nil {
log.Fatalf("Unable to create data operations client. %v", err)
}
http.Handle("/", appHandler(mainHandler))
appengine.Main() // Never returns.
}
func main() {
flag.Usage = func() {
fmt.Printf("Usage: scantest [options] <table_name>\n\n")
flag.PrintDefaults()
}
var err error
config, err = cbtconfig.Load()
if err != nil {
log.Fatal(err)
}
config.RegisterFlags()
flag.Parse()
if err := config.CheckFlags(cbtconfig.ProjectAndInstanceRequired); err != nil {
log.Fatal(err)
}
if config.Creds != "" {
os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", config.Creds)
}
if flag.NArg() != 1 {
flag.Usage()
os.Exit(1)
}
table := flag.Arg(0)
log.Printf("Dialing connections...")
client, err = bigtable.NewClient(context.Background(), config.Project, config.Instance)
if err != nil {
log.Fatalf("Making bigtable.Client: %v", err)
}
defer client.Close()
log.Printf("Starting scan test... (run for %v)", *runFor)
tbl := client.Open(table)
sem := make(chan int, *numScans) // limit the number of requests happening at once
var scans stats
stopTime := time.Now().Add(*runFor)
var wg sync.WaitGroup
for time.Now().Before(stopTime) {
sem <- 1
wg.Add(1)
go func() {
defer wg.Done()
defer func() { <-sem }()
ok := true
opStart := time.Now()
defer func() {
scans.Record(ok, time.Since(opStart))
}()
// Start at a random row key
key := fmt.Sprintf("user%d", rand.Int63())
limit := bigtable.LimitRows(int64(*rowLimit))
noop := func(bigtable.Row) bool { return true }
if err := tbl.ReadRows(context.Background(), bigtable.NewRange(key, ""), noop, limit); err != nil {
log.Printf("Error during scan: %v", err)
ok = false
}
}()
}
wg.Wait()
agg := stat.NewAggregate("scans", scans.ds, scans.tries-scans.ok)
log.Printf("Scans (%d ok / %d tries):\nscan times:\n%v\nthroughput (rows/second):\n%v",
scans.ok, scans.tries, agg, throughputString(agg))
}
func main() {
var err error
config, err = cbtrc.Load()
if err != nil {
log.Fatal(err)
}
config.RegisterFlags()
flag.Parse()
if err := config.CheckFlags(); err != nil {
log.Fatal(err)
}
if config.Creds != "" {
os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", config.Creds)
}
if flag.NArg() != 0 {
flag.Usage()
os.Exit(1)
}
var options []option.ClientOption
if *poolSize > 1 {
options = append(options, option.WithGRPCConnectionPool(*poolSize))
}
var csvFile *os.File
if *csvOutput != "" {
csvFile, err = os.Create(*csvOutput)
if err != nil {
log.Fatalf("creating csv output file: %v", err)
}
defer csvFile.Close()
log.Printf("Writing statistics to %q ...", *csvOutput)
}
log.Printf("Dialing connections...")
client, err = bigtable.NewClient(context.Background(), config.Project, config.Instance, options...)
if err != nil {
log.Fatalf("Making bigtable.Client: %v", err)
}
defer client.Close()
adminClient, err = bigtable.NewAdminClient(context.Background(), config.Project, config.Instance)
if err != nil {
log.Fatalf("Making bigtable.AdminClient: %v", err)
}
defer adminClient.Close()
// Create a scratch table.
log.Printf("Setting up scratch table...")
if err := adminClient.CreateTable(context.Background(), *scratchTable); err != nil {
log.Fatalf("Making scratch table %q: %v", *scratchTable, err)
}
if err := adminClient.CreateColumnFamily(context.Background(), *scratchTable, "f"); err != nil {
log.Fatalf("Making scratch table column family: %v", err)
}
// Upon a successful run, delete the table. Don't bother checking for errors.
defer adminClient.DeleteTable(context.Background(), *scratchTable)
log.Printf("Starting load test... (run for %v)", *runFor)
tbl := client.Open(*scratchTable)
sem := make(chan int, *reqCount) // limit the number of requests happening at once
var reads, writes stats
stopTime := time.Now().Add(*runFor)
var wg sync.WaitGroup
for time.Now().Before(stopTime) {
sem <- 1
wg.Add(1)
go func() {
defer wg.Done()
defer func() { <-sem }()
ok := true
opStart := time.Now()
var stats *stats
defer func() {
stats.Record(ok, time.Since(opStart))
}()
row := fmt.Sprintf("row%d", rand.Intn(100)) // operate on 1 of 100 rows
switch rand.Intn(10) {
default:
// read
stats = &reads
_, err := tbl.ReadRow(context.Background(), row, bigtable.RowFilter(bigtable.LatestNFilter(1)))
if err != nil {
log.Printf("Error doing read: %v", err)
ok = false
}
case 0, 1, 2, 3, 4:
// write
stats = &writes
mut := bigtable.NewMutation()
mut.Set("f", "col", bigtable.Now(), bytes.Repeat([]byte("0"), 1<<10)) // 1 KB write
if err := tbl.Apply(context.Background(), row, mut); err != nil {
log.Printf("Error doing mutation: %v", err)
ok = false
}
}
}()
}
wg.Wait()
readsAgg := stat.NewAggregate("reads", reads.ds, reads.tries-reads.ok)
writesAgg := stat.NewAggregate("writes", writes.ds, writes.tries-writes.ok)
log.Printf("Reads (%d ok / %d tries):\n%v", reads.ok, reads.tries, readsAgg)
log.Printf("Writes (%d ok / %d tries):\n%v", writes.ok, writes.tries, writesAgg)
if csvFile != nil {
stat.WriteCSV([]*stat.Aggregate{readsAgg, writesAgg}, csvFile)
}
}
func main() {
project := flag.String("project", "", "The Google Cloud Platform project ID. Required.")
instance := flag.String("instance", "", "The Google Cloud Bigtable instance ID. Required.")
flag.Parse()
for _, f := range []string{"project", "instance"} {
if flag.Lookup(f).Value.String() == "" {
log.Fatalf("The %s flag is required.", f)
}
}
ctx := context.Background()
// Set up admin client, tables, and column families.
// NewAdminClient uses Application Default Credentials to authenticate.
adminClient, err := bigtable.NewAdminClient(ctx, *project, *instance)
if err != nil {
log.Fatalf("Could not create admin client: %v", err)
}
tables, err := adminClient.Tables(ctx)
if err != nil {
log.Fatalf("Could not fetch table list: %v", err)
}
if !sliceContains(tables, tableName) {
log.Printf("Creating table %s", tableName)
if err := adminClient.CreateTable(ctx, tableName); err != nil {
log.Fatalf("Could not create table %s: %v", tableName, err)
}
}
tblInfo, err := adminClient.TableInfo(ctx, tableName)
if err != nil {
log.Fatalf("Could not read info for table %s: %v", tableName, err)
}
if !sliceContains(tblInfo.Families, columnFamilyName) {
if err := adminClient.CreateColumnFamily(ctx, tableName, columnFamilyName); err != nil {
log.Fatalf("Could not create column family %s: %v", columnFamilyName, err)
}
}
// Set up Bigtable data operations client.
// NewClient uses Application Default Credentials to authenticate.
client, err := bigtable.NewClient(ctx, *project, *instance)
if err != nil {
log.Fatalf("Could not create data operations client: %v", err)
}
tbl := client.Open(tableName)
muts := make([]*bigtable.Mutation, len(greetings))
rowKeys := make([]string, len(greetings))
log.Printf("Writing greeting rows to table")
for i, greeting := range greetings {
muts[i] = bigtable.NewMutation()
muts[i].Set(columnFamilyName, columnName, bigtable.Now(), []byte(greeting))
// Each row has a unique row key.
//
// Note: This example uses sequential numeric IDs for simplicity, but
// this can result in poor performance in a production application.
// Since rows are stored in sorted order by key, sequential keys can
// result in poor distribution of operations across nodes.
//
// For more information about how to design a Bigtable schema for the
// best performance, see the documentation:
//
// https://cloud.google.com/bigtable/docs/schema-design
rowKeys[i] = fmt.Sprintf("%s%d", columnName, i)
}
rowErrs, err := tbl.ApplyBulk(ctx, rowKeys, muts)
if err != nil {
log.Fatalf("Could not apply bulk row mutation: %v", err)
}
if rowErrs != nil {
for _, rowErr := range rowErrs {
log.Printf("Error writing row: %v", rowErr)
}
log.Fatalf("Could not write some rows")
}
log.Printf("Getting a single greeting by row key:")
row, err := tbl.ReadRow(ctx, rowKeys[0], bigtable.RowFilter(bigtable.ColumnFilter(columnName)))
if err != nil {
log.Fatalf("Could not read row with key %s: %v", rowKeys[0], err)
}
log.Printf("\t%s = %s\n", rowKeys[0], string(row[columnFamilyName][0].Value))
log.Printf("Reading all greeting rows:")
err = tbl.ReadRows(ctx, bigtable.PrefixRange(columnName), func(row bigtable.Row) bool {
item := row[columnFamilyName][0]
log.Printf("\t%s = %s\n", item.Row, string(item.Value))
return true
}, bigtable.RowFilter(bigtable.ColumnFilter(columnName)))
if err = client.Close(); err != nil {
log.Fatalf("Could not close data operations client: %v", err)
}
log.Printf("Deleting the table")
if err = adminClient.DeleteTable(ctx, tableName); err != nil {
log.Fatalf("Could not delete table %s: %v", tableName, err)
}
if err = adminClient.Close(); err != nil {
log.Fatalf("Could not close admin client: %v", err)
}
}