Example #1
0
func main() {
	var (
		project   = flag.String("project", "", "The name of the project.")
		instance  = flag.String("instance", "", "The name of the Cloud Bigtable instance.")
		tableName = flag.String("table", "docindex", "The name of the table containing the documents and index.")
		port      = flag.Int("port", 8080, "TCP port for server.")
	)
	flag.Parse()

	// Make an admin client.
	adminClient, err := bigtable.NewAdminClient(context.Background(), *project, *instance)
	if err != nil {
		log.Fatal("Bigtable NewAdminClient:", err)
	}

	// Make a regular client.
	client, err := bigtable.NewClient(context.Background(), *project, *instance)
	if err != nil {
		log.Fatal("Bigtable NewClient:", err)
	}

	// Open the table.
	table := client.Open(*tableName)

	// Set up HTML handlers, and start the web server.
	http.HandleFunc("/search", func(w http.ResponseWriter, r *http.Request) { handleSearch(w, r, table) })
	http.HandleFunc("/content", func(w http.ResponseWriter, r *http.Request) { handleContent(w, r, table) })
	http.HandleFunc("/add", func(w http.ResponseWriter, r *http.Request) { handleAddDoc(w, r, table) })
	http.HandleFunc("/reset", func(w http.ResponseWriter, r *http.Request) { handleReset(w, r, *tableName, adminClient) })
	http.HandleFunc("/copy", func(w http.ResponseWriter, r *http.Request) { handleCopy(w, r, *tableName, client, adminClient) })
	http.HandleFunc("/", handleMain)
	log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*port), nil))
}
Example #2
0
func getClient() *bigtable.Client {
	if client == nil {
		var err error
		client, err = bigtable.NewClient(context.Background(), config.Project, config.Zone, config.Cluster)
		if err != nil {
			log.Fatalf("Making bigtable.Client: %v", err)
		}
	}
	return client
}
func OpenClient(ctx context.Context, connectionData ClientConnectionData) *bigtable.Client {

	clientOpt := getClientOptionFromJsonKeyFile(ctx, connectionData.KeyJsonFilePath, bigtable.Scope)

	client, err := bigtable.NewClient(ctx, connectionData.Project, connectionData.Zone, connectionData.Cluster, clientOpt)

	if err != nil {
		fmt.Println("Error on [NewClient]: %v", err)
	}

	return client
}
Example #4
0
func main() {
	flag.Parse()
	ctx := context.Background()
	os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "grpc-go16-dev-608d72c9bce0.json")
	const tableName = "mytable"
	if *create {
		ac, err := bigtable.NewAdminClient(ctx, "grpc-go16-dev", "us-central1-c", "dollar-95-per-hour")
		if err != nil {
			log.Fatalf("AdminClient: %v", err)
		}
		if err := ac.CreateTable(ctx, tableName); err != nil {
			log.Printf("Create: %v", err)
		}
		if err := ac.CreateColumnFamily(ctx, tableName, "fam"); err != nil {
			log.Printf("CreateColumnFamily: %v", err)
		}
	}

	bc, err := bigtable.NewClient(ctx, "grpc-go16-dev", "us-central1-c", "dollar-95-per-hour")
	if err != nil {
		log.Fatalf("NewClient: %v", err)
	}
	t := bc.Open(tableName)
	m := bigtable.NewMutation()
	m.Set("fam", "col", bigtable.Now(), []byte("hello"))

	if *insert {
		log.Printf("Apply...")
		err = t.Apply(ctx, fmt.Sprintf("t:%v", time.Now().Unix()), m)
		if err != nil {
			log.Fatalf("Apply: %v", err)
		}
	}

	log.Printf("ReadRows...")

	t.ReadRows(ctx, bigtable.InfiniteRange(""), func(row bigtable.Row) bool {
		log.Printf("Row: %v", row.Key())
		for fam, ris := range row {
			log.Printf("  col fam %q", fam)
			for _, ri := range ris {
				log.Printf("    row %q, col %q, ts %v, val %q", ri.Row, ri.Column, ri.Timestamp, ri.Value)
			}
		}
		return true
	})
	log.Printf("done.")
}
Example #5
0
func main() {
	flag.Parse()

	if *tableName == prototypeTableName {
		log.Fatal("Can't use " + prototypeTableName + " as your table.")
	}

	// Let the library get credentials from file.
	os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", *credFile)

	// Make an admin client.
	var err error
	if adminClient, err = bigtable.NewAdminClient(context.Background(), *project, *zone, *cluster); err != nil {
		log.Fatal("Bigtable NewAdminClient:", err)
	}

	// Make a regular client.
	client, err = bigtable.NewClient(context.Background(), *project, *zone, *cluster)
	if err != nil {
		log.Fatal("Bigtable NewClient:", err)
	}

	// Open the table.
	table = client.Open(*tableName)

	// Rebuild the table if the command-line flag is set.
	if *rebuild {
		if err := rebuildTable(); err != nil {
			log.Fatal(err)
		}
	}

	// Set up HTML handlers, and start the web server.
	http.HandleFunc("/search", handleSearch)
	http.HandleFunc("/content", handleContent)
	http.HandleFunc("/add", handleAddDoc)
	http.HandleFunc("/clearindex", handleClear)
	http.HandleFunc("/", handleMain)
	log.Fatal(http.ListenAndServe(":8080", nil))
}
Example #6
0
func main() {
	ctx := context.Background()

	// Set up admin client, tables, and column families.
	// NewAdminClient uses Application Default Credentials to authenticate.
	adminClient, err := bigtable.NewAdminClient(ctx, project, zone, cluster)
	if err != nil {
		log.Fatalf("Unable to create a table admin client. %v", err)
	}
	tables, err := adminClient.Tables(ctx)
	if err != nil {
		log.Fatalf("Unable to fetch table list. %v", err)
	}
	if !sliceContains(tables, tableName) {
		if err := adminClient.CreateTable(ctx, tableName); err != nil {
			log.Fatalf("Unable to create table: %v. %v", tableName, err)
		}
	}
	tblInfo, err := adminClient.TableInfo(ctx, tableName)
	if err != nil {
		log.Fatalf("Unable to read info for table: %v. %v", tableName, err)
	}
	if !sliceContains(tblInfo.Families, familyName) {
		if err := adminClient.CreateColumnFamily(ctx, tableName, familyName); err != nil {
			log.Fatalf("Unable to create column family: %v. %v", familyName, err)
		}
	}
	adminClient.Close()

	// Set up Bigtable data operations client.
	// NewClient uses Application Default Credentials to authenticate.
	client, err = bigtable.NewClient(ctx, project, zone, cluster)
	if err != nil {
		log.Fatalf("Unable to create data operations client. %v", err)
	}

	http.Handle("/", appHandler(mainHandler))
	appengine.Main() // Never returns.
}
Example #7
0
func main() {
	flag.Usage = func() {
		fmt.Printf("Usage: scantest [options] <table_name>\n\n")
		flag.PrintDefaults()
	}

	var err error
	config, err = cbtrc.Load()
	if err != nil {
		log.Fatal(err)
	}
	config.RegisterFlags()

	flag.Parse()
	if err := config.CheckFlags(); err != nil {
		log.Fatal(err)
	}
	if config.Creds != "" {
		os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", config.Creds)
	}
	if flag.NArg() != 1 {
		flag.Usage()
		os.Exit(1)
	}

	table := flag.Arg(0)

	log.Printf("Dialing connections...")
	client, err = bigtable.NewClient(context.Background(), config.Project, config.Instance)
	if err != nil {
		log.Fatalf("Making bigtable.Client: %v", err)
	}
	defer client.Close()

	log.Printf("Starting scan test... (run for %v)", *runFor)
	tbl := client.Open(table)
	sem := make(chan int, *numScans) // limit the number of requests happening at once
	var scans stats

	stopTime := time.Now().Add(*runFor)
	var wg sync.WaitGroup
	for time.Now().Before(stopTime) {
		sem <- 1
		wg.Add(1)
		go func() {
			defer wg.Done()
			defer func() { <-sem }()

			ok := true
			opStart := time.Now()
			defer func() {
				scans.Record(ok, time.Since(opStart))
			}()

			// Start at a random row key
			key := fmt.Sprintf("user%d", rand.Int63())
			limit := bigtable.LimitRows(int64(*rowLimit))
			noop := func(bigtable.Row) bool { return true }
			if err := tbl.ReadRows(context.Background(), bigtable.NewRange(key, ""), noop, limit); err != nil {
				log.Printf("Error during scan: %v", err)
				ok = false
			}
		}()
	}
	wg.Wait()

	agg := stat.NewAggregate("scans", scans.ds, scans.tries-scans.ok)
	log.Printf("Scans (%d ok / %d tries):\nscan times:\n%v\nthroughput (rows/second):\n%v",
		scans.ok, scans.tries, agg, throughputString(agg))
}
Example #8
0
func main() {
	var err error
	config, err = cbtrc.Load()
	if err != nil {
		log.Fatal(err)
	}
	config.RegisterFlags()

	flag.Parse()
	if err := config.CheckFlags(); err != nil {
		log.Fatal(err)
	}
	if config.Creds != "" {
		os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", config.Creds)
	}
	if flag.NArg() != 0 {
		flag.Usage()
		os.Exit(1)
	}

	log.Printf("Dialing connections...")
	client, err = bigtable.NewClient(context.Background(), config.Project, config.Zone, config.Cluster)
	if err != nil {
		log.Fatalf("Making bigtable.Client: %v", err)
	}
	defer client.Close()
	adminClient, err = bigtable.NewAdminClient(context.Background(), config.Project, config.Zone, config.Cluster)
	if err != nil {
		log.Fatalf("Making bigtable.AdminClient: %v", err)
	}
	defer adminClient.Close()

	// Create a scratch table.
	log.Printf("Setting up scratch table...")
	if err := adminClient.CreateTable(context.Background(), *scratchTable); err != nil {
		log.Fatalf("Making scratch table %q: %v", *scratchTable, err)
	}
	if err := adminClient.CreateColumnFamily(context.Background(), *scratchTable, "f"); err != nil {
		log.Fatalf("Making scratch table column family: %v", err)
	}
	// Upon a successful run, delete the table. Don't bother checking for errors.
	defer adminClient.DeleteTable(context.Background(), *scratchTable)

	log.Printf("Starting load test... (run for %v)", *runFor)
	tbl := client.Open(*scratchTable)
	sem := make(chan int, 100) // limit the number of requests happening at once
	var reads, writes stats
	stopTime := time.Now().Add(*runFor)
	var wg sync.WaitGroup
	for time.Now().Before(stopTime) {
		sem <- 1
		wg.Add(1)
		go func() {
			defer wg.Done()
			defer func() { <-sem }()

			ok := true
			opStart := time.Now()
			var stats *stats
			defer func() {
				stats.Record(ok, time.Since(opStart))
			}()

			row := fmt.Sprintf("row%d", rand.Intn(100)) // operate on 1 of 100 rows

			switch rand.Intn(10) {
			default:
				// read
				stats = &reads
				_, err := tbl.ReadRow(context.Background(), row, bigtable.RowFilter(bigtable.LatestNFilter(1)))
				if err != nil {
					log.Printf("Error doing read: %v", err)
					ok = false
				}
			case 0, 1, 2, 3, 4:
				// write
				stats = &writes
				mut := bigtable.NewMutation()
				mut.Set("f", "col", bigtable.Now(), bytes.Repeat([]byte("0"), 1<<10)) // 1 KB write
				if err := tbl.Apply(context.Background(), row, mut); err != nil {
					log.Printf("Error doing mutation: %v", err)
					ok = false
				}
			}
		}()
	}
	wg.Wait()

	log.Printf("Reads (%d ok / %d tries):\n%v", reads.ok, reads.tries, newAggregate(reads.ds))
	log.Printf("Writes (%d ok / %d tries):\n%v", writes.ok, writes.tries, newAggregate(writes.ds))
}
Example #9
0
File: main.go Project: abema/cloth
func main() {

	// test struct
	user := User{
		ID:         "vytxeTZskVKR7C7WgdSP3d",
		Name:       "osamingo",
		Content:    "WRYYY!",
		Purchased:  true,
		LoggedInAt: time.Now().Unix(),
		CreatedAt:  time.Now().Add(-24 * time.Hour).Unix(),
	}

	// dialing to test server
	conn, err := grpc.Dial(srv.Addr, grpc.WithInsecure())
	if err != nil {
		fmt.Println(err)
		return
	}

	// generate a client
	client, err := bigtable.NewClient(context.Background(), project, zone, cluster, cloud.WithBaseGRPC(conn))
	if err != nil {
		fmt.Println(err)
		return
	}
	defer client.Close()

	// generate Mutation
	mutation, err := cloth.GenerateColumnsMutation(cf, time.Now(), &user)
	if err != nil {
		fmt.Println(err)
		return
	}

	// apply
	err = client.Open(tbl).Apply(context.Background(), user.ID, mutation)
	if err != nil {
		fmt.Println(err)
		return
	}

	// read a row
	row, err := client.Open(tbl).ReadRow(context.Background(), user.ID)
	if err != nil {
		fmt.Println(err)
		return
	}

	// convert
	target := User{}
	err = cloth.ReadItems(row[cf], &target)
	if err != nil {
		fmt.Println(err)
		return
	}

	// check
	if target.ID == user.ID {
		fmt.Println("ID:\t\tOK")
	}
	if target.Name == user.Name {
		fmt.Println("Name:\t\tOK")
	}
	if target.Content == user.Content {
		fmt.Println("Content:\tOK")
	}
	if target.Purchased == user.Purchased {
		fmt.Println("Purchased:\tOK")
	}
	if target.LoggedInAt == 0 {
		fmt.Println("LoggedInAt:\tOK (shoud be zero value)")
	}
	if target.CreatedAt == user.CreatedAt {
		fmt.Println("CreatedAt:\tOK")
	}
}