func init() {
var err error
srv, err = bttest.NewServer()
if err != nil {
panic(err)
}
conn, err := grpc.Dial(srv.Addr, grpc.WithInsecure())
if err != nil {
panic(err)
}
ctx := context.Background()
admin, err := bigtable.NewAdminClient(ctx, project, zone, cluster, cloud.WithBaseGRPC(conn))
if err != nil {
panic(err)
}
defer admin.Close()
err = admin.CreateTable(ctx, tbl)
if err != nil {
panic(err)
}
err = admin.CreateColumnFamily(ctx, tbl, cf)
if err != nil {
panic(err)
}
}
func main() {
var (
project = flag.String("project", "", "The name of the project.")
instance = flag.String("instance", "", "The name of the Cloud Bigtable instance.")
tableName = flag.String("table", "docindex", "The name of the table containing the documents and index.")
port = flag.Int("port", 8080, "TCP port for server.")
)
flag.Parse()
// Make an admin client.
adminClient, err := bigtable.NewAdminClient(context.Background(), *project, *instance)
if err != nil {
log.Fatal("Bigtable NewAdminClient:", err)
}
// Make a regular client.
client, err := bigtable.NewClient(context.Background(), *project, *instance)
if err != nil {
log.Fatal("Bigtable NewClient:", err)
}
// Open the table.
table := client.Open(*tableName)
// Set up HTML handlers, and start the web server.
http.HandleFunc("/search", func(w http.ResponseWriter, r *http.Request) { handleSearch(w, r, table) })
http.HandleFunc("/content", func(w http.ResponseWriter, r *http.Request) { handleContent(w, r, table) })
http.HandleFunc("/add", func(w http.ResponseWriter, r *http.Request) { handleAddDoc(w, r, table) })
http.HandleFunc("/reset", func(w http.ResponseWriter, r *http.Request) { handleReset(w, r, *tableName, adminClient) })
http.HandleFunc("/copy", func(w http.ResponseWriter, r *http.Request) { handleCopy(w, r, *tableName, client, adminClient) })
http.HandleFunc("/", handleMain)
log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*port), nil))
}
func getAdminClient() *bigtable.AdminClient {
if adminClient == nil {
var err error
adminClient, err = bigtable.NewAdminClient(context.Background(), config.Project, config.Zone, config.Cluster)
if err != nil {
log.Fatalf("Making bigtable.AdminClient: %v", err)
}
}
return adminClient
}
func OpenAdminClient(ctx context.Context, connectionData ClientConnectionData) *bigtable.AdminClient {
clientOpt := getClientOptionFromJsonKeyFile(ctx, connectionData.KeyJsonFilePath, bigtable.AdminScope)
client, err := bigtable.NewAdminClient(ctx, connectionData.Project, connectionData.Zone, connectionData.Cluster, clientOpt)
if err != nil {
fmt.Println("Error on [NewAdminClient]: %v", err)
}
return client
}
func main() {
flag.Parse()
ctx := context.Background()
os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "grpc-go16-dev-608d72c9bce0.json")
const tableName = "mytable"
if *create {
ac, err := bigtable.NewAdminClient(ctx, "grpc-go16-dev", "us-central1-c", "dollar-95-per-hour")
if err != nil {
log.Fatalf("AdminClient: %v", err)
}
if err := ac.CreateTable(ctx, tableName); err != nil {
log.Printf("Create: %v", err)
}
if err := ac.CreateColumnFamily(ctx, tableName, "fam"); err != nil {
log.Printf("CreateColumnFamily: %v", err)
}
}
bc, err := bigtable.NewClient(ctx, "grpc-go16-dev", "us-central1-c", "dollar-95-per-hour")
if err != nil {
log.Fatalf("NewClient: %v", err)
}
t := bc.Open(tableName)
m := bigtable.NewMutation()
m.Set("fam", "col", bigtable.Now(), []byte("hello"))
if *insert {
log.Printf("Apply...")
err = t.Apply(ctx, fmt.Sprintf("t:%v", time.Now().Unix()), m)
if err != nil {
log.Fatalf("Apply: %v", err)
}
}
log.Printf("ReadRows...")
t.ReadRows(ctx, bigtable.InfiniteRange(""), func(row bigtable.Row) bool {
log.Printf("Row: %v", row.Key())
for fam, ris := range row {
log.Printf(" col fam %q", fam)
for _, ri := range ris {
log.Printf(" row %q, col %q, ts %v, val %q", ri.Row, ri.Column, ri.Timestamp, ri.Value)
}
}
return true
})
log.Printf("done.")
}
func adminClient(addr string) (*bgt.AdminClient, error) {
conn, err := grpc.Dial(addr, grpc.WithInsecure())
if err != nil {
return nil, err
}
ctx, _ := context.WithTimeout(context.Background(), defaultTimeout)
adminClient, err := bgt.NewAdminClient(
ctx,
"proj",
"zone",
"cluster",
cloud.WithBaseGRPC(conn))
if err != nil {
return nil, err
}
return adminClient, nil
}
func main() {
flag.Parse()
if *tableName == prototypeTableName {
log.Fatal("Can't use " + prototypeTableName + " as your table.")
}
// Let the library get credentials from file.
os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", *credFile)
// Make an admin client.
var err error
if adminClient, err = bigtable.NewAdminClient(context.Background(), *project, *zone, *cluster); err != nil {
log.Fatal("Bigtable NewAdminClient:", err)
}
// Make a regular client.
client, err = bigtable.NewClient(context.Background(), *project, *zone, *cluster)
if err != nil {
log.Fatal("Bigtable NewClient:", err)
}
// Open the table.
table = client.Open(*tableName)
// Rebuild the table if the command-line flag is set.
if *rebuild {
if err := rebuildTable(); err != nil {
log.Fatal(err)
}
}
// Set up HTML handlers, and start the web server.
http.HandleFunc("/search", handleSearch)
http.HandleFunc("/content", handleContent)
http.HandleFunc("/add", handleAddDoc)
http.HandleFunc("/clearindex", handleClear)
http.HandleFunc("/", handleMain)
log.Fatal(http.ListenAndServe(":8080", nil))
}
func main() {
ctx := context.Background()
// Set up admin client, tables, and column families.
// NewAdminClient uses Application Default Credentials to authenticate.
adminClient, err := bigtable.NewAdminClient(ctx, project, zone, cluster)
if err != nil {
log.Fatalf("Unable to create a table admin client. %v", err)
}
tables, err := adminClient.Tables(ctx)
if err != nil {
log.Fatalf("Unable to fetch table list. %v", err)
}
if !sliceContains(tables, tableName) {
if err := adminClient.CreateTable(ctx, tableName); err != nil {
log.Fatalf("Unable to create table: %v. %v", tableName, err)
}
}
tblInfo, err := adminClient.TableInfo(ctx, tableName)
if err != nil {
log.Fatalf("Unable to read info for table: %v. %v", tableName, err)
}
if !sliceContains(tblInfo.Families, familyName) {
if err := adminClient.CreateColumnFamily(ctx, tableName, familyName); err != nil {
log.Fatalf("Unable to create column family: %v. %v", familyName, err)
}
}
adminClient.Close()
// Set up Bigtable data operations client.
// NewClient uses Application Default Credentials to authenticate.
client, err = bigtable.NewClient(ctx, project, zone, cluster)
if err != nil {
log.Fatalf("Unable to create data operations client. %v", err)
}
http.Handle("/", appHandler(mainHandler))
appengine.Main() // Never returns.
}
func main() {
var err error
config, err = cbtrc.Load()
if err != nil {
log.Fatal(err)
}
config.RegisterFlags()
flag.Parse()
if err := config.CheckFlags(); err != nil {
log.Fatal(err)
}
if config.Creds != "" {
os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", config.Creds)
}
if flag.NArg() != 0 {
flag.Usage()
os.Exit(1)
}
log.Printf("Dialing connections...")
client, err = bigtable.NewClient(context.Background(), config.Project, config.Zone, config.Cluster)
if err != nil {
log.Fatalf("Making bigtable.Client: %v", err)
}
defer client.Close()
adminClient, err = bigtable.NewAdminClient(context.Background(), config.Project, config.Zone, config.Cluster)
if err != nil {
log.Fatalf("Making bigtable.AdminClient: %v", err)
}
defer adminClient.Close()
// Create a scratch table.
log.Printf("Setting up scratch table...")
if err := adminClient.CreateTable(context.Background(), *scratchTable); err != nil {
log.Fatalf("Making scratch table %q: %v", *scratchTable, err)
}
if err := adminClient.CreateColumnFamily(context.Background(), *scratchTable, "f"); err != nil {
log.Fatalf("Making scratch table column family: %v", err)
}
// Upon a successful run, delete the table. Don't bother checking for errors.
defer adminClient.DeleteTable(context.Background(), *scratchTable)
log.Printf("Starting load test... (run for %v)", *runFor)
tbl := client.Open(*scratchTable)
sem := make(chan int, 100) // limit the number of requests happening at once
var reads, writes stats
stopTime := time.Now().Add(*runFor)
var wg sync.WaitGroup
for time.Now().Before(stopTime) {
sem <- 1
wg.Add(1)
go func() {
defer wg.Done()
defer func() { <-sem }()
ok := true
opStart := time.Now()
var stats *stats
defer func() {
stats.Record(ok, time.Since(opStart))
}()
row := fmt.Sprintf("row%d", rand.Intn(100)) // operate on 1 of 100 rows
switch rand.Intn(10) {
default:
// read
stats = &reads
_, err := tbl.ReadRow(context.Background(), row, bigtable.RowFilter(bigtable.LatestNFilter(1)))
if err != nil {
log.Printf("Error doing read: %v", err)
ok = false
}
case 0, 1, 2, 3, 4:
// write
stats = &writes
mut := bigtable.NewMutation()
mut.Set("f", "col", bigtable.Now(), bytes.Repeat([]byte("0"), 1<<10)) // 1 KB write
if err := tbl.Apply(context.Background(), row, mut); err != nil {
log.Printf("Error doing mutation: %v", err)
ok = false
}
}
}()
}
wg.Wait()
log.Printf("Reads (%d ok / %d tries):\n%v", reads.ok, reads.tries, newAggregate(reads.ds))
log.Printf("Writes (%d ok / %d tries):\n%v", writes.ok, writes.tries, newAggregate(writes.ds))
}