func main() {
flag.Parse()
flagsOk := true
if flag.Lookup("project").Value.String() == "" {
fmt.Fprintf(os.Stderr, "Flag --project is required\n")
flagsOk = false
}
var sourceFlagCount int
if flag.Lookup("dataset").Value.String() != "" {
sourceFlagCount++
}
if flag.Lookup("jobid").Value.String() != "" {
sourceFlagCount++
}
if sourceFlagCount != 1 {
fmt.Fprintf(os.Stderr, "Exactly one of --dataset or --jobid must be set\n")
flagsOk = false
}
if !flagsOk {
os.Exit(1)
}
tableRE, err := regexp.Compile(*table)
if err != nil {
fmt.Fprintf(os.Stderr, "--table is not a valid regular expression: %q\n", *table)
os.Exit(1)
}
httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope)
if err != nil {
log.Fatalf("Creating http client: %v", err)
}
client, err := bigquery.NewClient(httpClient, *project)
if err != nil {
log.Fatalf("Creating bigquery client: %v", err)
}
if *jobID != "" {
printQueryResults(client, *jobID)
return
}
ds := client.Dataset(*dataset)
var tables []*bigquery.Table
tables, err = ds.ListTables(context.Background())
if err != nil {
log.Fatalf("Listing tables: %v", err)
}
for _, t := range tables {
if tableRE.MatchString(t.TableID) {
printTable(client, t)
}
}
}
func printQueryResults(client *bigquery.Client, queryJobID string) {
job, err := client.JobFromID(context.Background(), queryJobID)
if err != nil {
log.Fatalf("Loading job: %v", err)
}
it, err := client.Read(context.Background(), job)
if err != nil {
log.Fatalf("Reading: %v", err)
}
// TODO: print schema.
printValues(it)
}
// rebuildTable deletes the table if it exists, then creates the table, with the index column family.
func rebuildTable() error {
ctx, _ := context.WithTimeout(context.Background(), 5*time.Minute)
adminClient.DeleteTable(ctx, *tableName)
if err := adminClient.CreateTable(ctx, *tableName); err != nil {
return fmt.Errorf("CreateTable: %v", err)
}
time.Sleep(20 * time.Second)
if err := adminClient.CreateColumnFamily(ctx, *tableName, indexColumnFamily); err != nil {
return fmt.Errorf("CreateColumnFamily: %v", err)
}
if err := adminClient.CreateColumnFamily(ctx, *tableName, contentColumnFamily); err != nil {
return fmt.Errorf("CreateColumnFamily: %v", err)
}
// Open the prototype table. It contains a number of documents to get started with.
prototypeTable := client.Open(prototypeTableName)
var (
writeErr error // Set if any write fails.
mu sync.Mutex // Protects writeErr
wg sync.WaitGroup // Used to wait for all writes to finish.
)
copyRowToTable := func(row bigtable.Row) bool {
mu.Lock()
failed := writeErr != nil
mu.Unlock()
if failed {
return false
}
mut := bigtable.NewMutation()
for family, items := range row {
for _, item := range items {
// Get the column name, excluding the column family name and ':' character.
columnWithoutFamily := item.Column[len(family)+1:]
mut.Set(family, columnWithoutFamily, bigtable.Now(), item.Value)
}
}
wg.Add(1)
go func() {
// TODO: should use a semaphore to limit the number of concurrent writes.
if err := table.Apply(ctx, row.Key(), mut); err != nil {
mu.Lock()
writeErr = err
mu.Unlock()
}
wg.Done()
}()
return true
}
// Create a filter that only accepts the column families we're interested in.
filter := bigtable.FamilyFilter(indexColumnFamily + "|" + contentColumnFamily)
// Read every row from prototypeTable, and call copyRowToTable to copy it to our table.
err := prototypeTable.ReadRows(ctx, bigtable.InfiniteRange(""), copyRowToTable, bigtable.RowFilter(filter))
wg.Wait()
if err != nil {
return err
}
return writeErr
}
// FakeSingleContext returns a context whose Call invocations will be serviced
// by f, which should be a function that has two arguments of the input and output
// protocol buffer type, and one error return.
func FakeSingleContext(t *testing.T, service, method string, f interface{}) context.Context {
fv := reflect.ValueOf(f)
if fv.Kind() != reflect.Func {
t.Fatal("not a function")
}
ft := fv.Type()
if ft.NumIn() != 2 || ft.NumOut() != 1 {
t.Fatalf("f has %d in and %d out, want 2 in and 1 out", ft.NumIn(), ft.NumOut())
}
for i := 0; i < 2; i++ {
at := ft.In(i)
if !at.Implements(protoMessageType) {
t.Fatalf("arg %d does not implement proto.Message", i)
}
}
if ft.Out(0) != errorType {
t.Fatalf("f's return is %v, want error", ft.Out(0))
}
s := &single{
t: t,
service: service,
method: method,
f: fv,
}
return internal.WithCallOverride(context.Background(), s.call)
}
// DoCancelAfterFirstResponse cancels the RPC after receiving the first message from the server.
func DoCancelAfterFirstResponse(tc testpb.TestServiceClient) {
ctx, cancel := context.WithCancel(context.Background())
stream, err := tc.FullDuplexCall(ctx)
if err != nil {
grpclog.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err)
}
respParam := []*testpb.ResponseParameters{
{
Size: proto.Int32(31415),
},
}
pl := clientNewPayload(testpb.PayloadType_COMPRESSABLE, 27182)
req := &testpb.StreamingOutputCallRequest{
ResponseType: testpb.PayloadType_COMPRESSABLE.Enum(),
ResponseParameters: respParam,
Payload: pl,
}
if err := stream.Send(req); err != nil {
grpclog.Fatalf("%v.Send(%v) = %v", stream, req, err)
}
if _, err := stream.Recv(); err != nil {
grpclog.Fatalf("%v.Recv() = %v", stream, err)
}
cancel()
if _, err := stream.Recv(); grpc.Code(err) != codes.Canceled {
grpclog.Fatalf("%v compleled with error code %d, want %d", stream, grpc.Code(err), codes.Canceled)
}
grpclog.Println("CancelAfterFirstResponse done")
}
// NewRemoteContext returns a context that gives access to the production
// APIs for the application at the given host. All communication will be
// performed over SSL unless the host is localhost.
func NewRemoteContext(host string, client *http.Client) (context.Context, error) {
// Add an appcfg header to outgoing requests.
t := client.Transport
if t == nil {
t = http.DefaultTransport
}
client.Transport = &headerAddingRoundTripper{t}
url := url.URL{
Scheme: "https",
Host: host,
Path: "/_ah/remote_api",
}
if host == "localhost" || strings.HasPrefix(host, "localhost:") {
url.Scheme = "http"
}
u := url.String()
appID, err := getAppID(client, u)
if err != nil {
return nil, fmt.Errorf("unable to contact server: %v", err)
}
rc := &remoteContext{
client: client,
url: u,
}
ctx := internal.WithCallOverride(context.Background(), rc.call)
ctx = internal.WithLogOverride(ctx, rc.logf)
ctx = internal.WithAppIDOverride(ctx, appID)
return ctx, nil
}
// DoPerRPCCreds performs a unary RPC with per RPC OAUTH2 token.
func DoPerRPCCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, oauthScope string) {
jsonKey := getServiceAccountJSONKey(serviceAccountKeyFile)
pl := clientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize)
req := &testpb.SimpleRequest{
ResponseType: testpb.PayloadType_COMPRESSABLE.Enum(),
ResponseSize: proto.Int32(int32(largeRespSize)),
Payload: pl,
FillUsername: proto.Bool(true),
FillOauthScope: proto.Bool(true),
}
token := GetToken(serviceAccountKeyFile, oauthScope)
kv := map[string]string{"authorization": token.TokenType + " " + token.AccessToken}
ctx := metadata.NewContext(context.Background(), metadata.MD{"authorization": []string{kv["authorization"]}})
reply, err := tc.UnaryCall(ctx, req)
if err != nil {
grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err)
}
user := reply.GetUsername()
scope := reply.GetOauthScope()
if !strings.Contains(string(jsonKey), user) {
grpclog.Fatalf("Got user name %q which is NOT a substring of %q.", user, jsonKey)
}
if !strings.Contains(oauthScope, scope) {
grpclog.Fatalf("Got OAuth scope %q which is NOT a substring of %q.", scope, oauthScope)
}
grpclog.Println("PerRPCCreds done")
}
// DoClientStreaming performs a client streaming RPC.
func DoClientStreaming(tc testpb.TestServiceClient) {
stream, err := tc.StreamingInputCall(context.Background())
if err != nil {
grpclog.Fatalf("%v.StreamingInputCall(_) = _, %v", tc, err)
}
var sum int
for _, s := range reqSizes {
pl := clientNewPayload(testpb.PayloadType_COMPRESSABLE, s)
req := &testpb.StreamingInputCallRequest{
Payload: pl,
}
if err := stream.Send(req); err != nil {
grpclog.Fatalf("%v.Send(%v) = %v", stream, req, err)
}
sum += s
grpclog.Printf("Sent a request of size %d, aggregated size %d", s, sum)
}
reply, err := stream.CloseAndRecv()
if err != nil {
grpclog.Fatalf("%v.CloseAndRecv() got error %v, want %v", stream, err, nil)
}
if reply.GetAggregatedPayloadSize() != int32(sum) {
grpclog.Fatalf("%v.CloseAndRecv().GetAggregatePayloadSize() = %v; want %v", stream, reply.GetAggregatedPayloadSize(), sum)
}
grpclog.Println("ClientStreaming done")
}
// handleContent fetches the content of a document from the Bigtable and returns it.
func handleContent(w http.ResponseWriter, r *http.Request) {
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
name := r.FormValue("name")
if len(name) == 0 {
http.Error(w, "No document name supplied.", http.StatusBadRequest)
return
}
row, err := table.ReadRow(ctx, name)
if err != nil {
http.Error(w, "Error reading content: "+err.Error(), http.StatusInternalServerError)
return
}
content := row[contentColumnFamily]
if len(content) == 0 {
http.Error(w, "Document not found.", http.StatusNotFound)
return
}
var buf bytes.Buffer
if err := contentTemplate.ExecuteTemplate(&buf, "", struct{ Title, Content string }{name, string(content[0].Value)}); err != nil {
http.Error(w, "Error executing HTML template: "+err.Error(), http.StatusInternalServerError)
return
}
io.Copy(w, &buf)
}
// printFeature gets the feature for the given point.
func printFeature(client pb.RouteGuideClient, point *pb.Point) {
grpclog.Printf("Getting feature for point (%d, %d)", point.Latitude, point.Longitude)
feature, err := client.GetFeature(context.Background(), point)
if err != nil {
grpclog.Fatalf("%v.GetFeatures(_) = _, %v: ", client, err)
}
grpclog.Println(feature)
}
func deleteTopic(client *pubsub.Client, argv []string) {
checkArgs(argv, 2)
topic := argv[1]
err := client.Topic(topic).Delete(context.Background())
if err != nil {
log.Fatalf("Deleting topic failed: %v", err)
}
fmt.Printf("Topic %s was deleted.\n", topic)
}
func checkTopicExists(client *pubsub.Client, argv []string) {
checkArgs(argv, 1)
topic := argv[1]
exists, err := client.Topic(topic).Exists(context.Background())
if err != nil {
log.Fatalf("Checking topic exists failed: %v", err)
}
fmt.Println(exists)
}
func createTopic(client *pubsub.Client, argv []string) {
checkArgs(argv, 2)
topic := argv[1]
_, err := client.NewTopic(context.Background(), topic)
if err != nil {
log.Fatalf("Creating topic failed: %v", err)
}
fmt.Printf("Topic %s was created.\n", topic)
}
func checkSubscriptionExists(client *pubsub.Client, argv []string) {
checkArgs(argv, 1)
sub := argv[1]
exists, err := client.Subscription(sub).Exists(context.Background())
if err != nil {
log.Fatalf("Checking subscription exists failed: %v", err)
}
fmt.Println(exists)
}
func deleteSubscription(client *pubsub.Client, argv []string) {
checkArgs(argv, 2)
sub := argv[1]
err := client.Subscription(sub).Delete(context.Background())
if err != nil {
log.Fatalf("Deleting Subscription failed: %v", err)
}
fmt.Printf("Subscription %s was deleted.\n", sub)
}
func getClusterAdminClient() *bigtable.ClusterAdminClient {
if clusterAdminClient == nil {
var err error
clusterAdminClient, err = bigtable.NewClusterAdminClient(context.Background(), config.Project)
if err != nil {
log.Fatalf("Making bigtable.ClusterAdminClient: %v", err)
}
}
return clusterAdminClient
}
func listTopics(client *pubsub.Client, argv []string) {
checkArgs(argv, 1)
topics, err := client.Topics(context.Background())
if err != nil {
log.Fatalf("Listing topics failed: %v", err)
}
for _, t := range topics {
fmt.Println(t.Name())
}
}
func printTable(client *bigquery.Client, t *bigquery.Table) {
it, err := client.Read(context.Background(), t)
if err != nil {
log.Fatalf("Reading: %v", err)
}
id := t.FullyQualifiedName()
fmt.Printf("%s\n%s\n", id, strings.Repeat("-", len(id)))
printValues(it)
}
// DoEmptyUnaryCall performs a unary RPC with empty request and response messages.
func DoEmptyUnaryCall(tc testpb.TestServiceClient) {
reply, err := tc.EmptyCall(context.Background(), &testpb.Empty{})
if err != nil {
grpclog.Fatal("/TestService/EmptyCall RPC failed: ", err)
}
if !proto.Equal(&testpb.Empty{}, reply) {
grpclog.Fatalf("/TestService/EmptyCall receives %v, want %v", reply, testpb.Empty{})
}
grpclog.Println("EmptyUnaryCall done")
}
func listSubscriptions(client *pubsub.Client, argv []string) {
checkArgs(argv, 1)
subs, err := client.Subscriptions(context.Background())
if err != nil {
log.Fatalf("Listing subscriptions failed: %v", err)
}
for _, s := range subs {
fmt.Println(s.Name())
}
}
func getAdminClient() *bigtable.AdminClient {
if adminClient == nil {
var err error
adminClient, err = bigtable.NewAdminClient(context.Background(), config.Project, config.Zone, config.Cluster)
if err != nil {
log.Fatalf("Making bigtable.AdminClient: %v", err)
}
}
return adminClient
}
func createSubscription(client *pubsub.Client, argv []string) {
checkArgs(argv, 3)
sub := argv[1]
topic := argv[2]
_, err := client.Topic(topic).Subscribe(context.Background(), sub, nil)
if err != nil {
log.Fatalf("Creating Subscription failed: %v", err)
}
fmt.Printf("Subscription %s was created.\n", sub)
}
// DoUnaryCall performs an unary RPC with given stub and request and response sizes.
func DoUnaryCall(tc testpb.TestServiceClient, reqSize, respSize int) {
pl := newPayload(testpb.PayloadType_COMPRESSABLE, reqSize)
req := &testpb.SimpleRequest{
ResponseType: pl.Type,
ResponseSize: int32(respSize),
Payload: pl,
}
if _, err := tc.UnaryCall(context.Background(), req); err != nil {
grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err)
}
}
// GetToken obtains an OAUTH token from the input.
func GetToken(serviceAccountKeyFile string, oauthScope string) *oauth2.Token {
jsonKey := getServiceAccountJSONKey(serviceAccountKeyFile)
config, err := google.JWTConfigFromJSON(jsonKey, oauthScope)
if err != nil {
grpclog.Fatalf("Failed to get the config: %v", err)
}
token, err := config.TokenSource(context.Background()).Token()
if err != nil {
grpclog.Fatalf("Failed to get the token: %v", err)
}
return token
}
func main() {
flag.Parse()
if *tableName == prototypeTableName {
log.Fatal("Can't use " + prototypeTableName + " as your table.")
}
// Let the library get credentials from file.
os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", *credFile)
// Make an admin client.
var err error
if adminClient, err = bigtable.NewAdminClient(context.Background(), *project, *zone, *cluster); err != nil {
log.Fatal("Bigtable NewAdminClient:", err)
}
// Make a regular client.
client, err = bigtable.NewClient(context.Background(), *project, *zone, *cluster)
if err != nil {
log.Fatal("Bigtable NewClient:", err)
}
// Open the table.
table = client.Open(*tableName)
// Rebuild the table if the command-line flag is set.
if *rebuild {
if err := rebuildTable(); err != nil {
log.Fatal(err)
}
}
// Set up HTML handlers, and start the web server.
http.HandleFunc("/search", handleSearch)
http.HandleFunc("/content", handleContent)
http.HandleFunc("/add", handleAddDoc)
http.HandleFunc("/clearindex", handleClear)
http.HandleFunc("/", handleMain)
log.Fatal(http.ListenAndServe(":8080", nil))
}
// DoEmptyStream sets up a bi-directional streaming with zero message.
func DoEmptyStream(tc testpb.TestServiceClient) {
stream, err := tc.FullDuplexCall(context.Background())
if err != nil {
grpclog.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err)
}
if err := stream.CloseSend(); err != nil {
grpclog.Fatalf("%v.CloseSend() got %v, want %v", stream, err, nil)
}
if _, err := stream.Recv(); err != io.EOF {
grpclog.Fatalf("%v failed to complete the empty stream test: %v", stream, err)
}
grpclog.Println("Emptystream done")
}
// DoCancelAfterBegin cancels the RPC after metadata has been sent but before payloads are sent.
func DoCancelAfterBegin(tc testpb.TestServiceClient) {
ctx, cancel := context.WithCancel(metadata.NewContext(context.Background(), testMetadata))
stream, err := tc.StreamingInputCall(ctx)
if err != nil {
grpclog.Fatalf("%v.StreamingInputCall(_) = _, %v", tc, err)
}
cancel()
_, err = stream.CloseAndRecv()
if grpc.Code(err) != codes.Canceled {
grpclog.Fatalf("%v.CloseAndRecv() got error code %d, want %d", stream, grpc.Code(err), codes.Canceled)
}
grpclog.Println("CancelAfterBegin done")
}
func closeLoopStream() {
s, conn, tc := buildConnection()
ch := make(chan int, *maxConcurrentRPCs*4)
var (
mu sync.Mutex
wg sync.WaitGroup
)
wg.Add(*maxConcurrentRPCs)
// Distribute RPCs over maxConcurrentCalls workers.
for i := 0; i < *maxConcurrentRPCs; i++ {
go func() {
stream, err := tc.StreamingCall(context.Background())
if err != nil {
grpclog.Fatalf("%v.StreamingCall(_) = _, %v", tc, err)
}
// Do some warm up.
for i := 0; i < 100; i++ {
streamCaller(tc, stream)
}
for range ch {
start := time.Now()
streamCaller(tc, stream)
elapse := time.Since(start)
mu.Lock()
s.Add(elapse)
mu.Unlock()
}
wg.Done()
}()
}
// Stop the client when time is up.
done := make(chan struct{})
go func() {
<-time.After(time.Duration(*duration) * time.Second)
close(done)
}()
ok := true
for ok {
select {
case ch <- 0:
case <-done:
ok = false
}
}
close(ch)
wg.Wait()
conn.Close()
grpclog.Println(s.String())
}
// printFeatures lists all the features within the given bounding Rectangle.
func printFeatures(client pb.RouteGuideClient, rect *pb.Rectangle) {
grpclog.Printf("Looking for features within %v", rect)
stream, err := client.ListFeatures(context.Background(), rect)
if err != nil {
grpclog.Fatalf("%v.ListFeatures(_) = _, %v", client, err)
}
for {
feature, err := stream.Recv()
if err == io.EOF {
break
}
if err != nil {
grpclog.Fatalf("%v.ListFeatures(_) = _, %v", client, err)
}
grpclog.Println(feature)
}
}
// DoLargeUnaryCall performs a unary RPC with large payload in the request and response.
func DoLargeUnaryCall(tc testpb.TestServiceClient) {
pl := clientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize)
req := &testpb.SimpleRequest{
ResponseType: testpb.PayloadType_COMPRESSABLE.Enum(),
ResponseSize: proto.Int32(int32(largeRespSize)),
Payload: pl,
}
reply, err := tc.UnaryCall(context.Background(), req)
if err != nil {
grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err)
}
t := reply.GetPayload().GetType()
s := len(reply.GetPayload().GetBody())
if t != testpb.PayloadType_COMPRESSABLE || s != largeRespSize {
grpclog.Fatalf("Got the reply with type %d len %d; want %d, %d", t, s, testpb.PayloadType_COMPRESSABLE, largeRespSize)
}
grpclog.Println("LargeUnaryCall done")
}