Ejemplo n.º 1
0
func GetNewStorageClient() (*StorageClient, error) {
	client, err := storage.NewClient(ctx, cloud.WithTokenSource(defaultTokenSource))
	return &StorageClient{
		Client: client,
		Ctx:    ctx,
	}, err
}
Ejemplo n.º 2
0
func cleanup(t *testing.T, prefix string) {
	ctx := context.Background()
	client, err := NewClient(ctx, cloud.WithTokenSource(testutil.TokenSource(ctx, ScopeFullControl)))
	if err != nil {
		t.Errorf("Could not create client: %v", err)
	}
	defer client.Close()

	var q *Query = &Query{
		Prefix: prefix,
	}
	for {
		o, err := client.Bucket(bucket).List(ctx, q)
		if err != nil {
			t.Fatalf("Cleanup List for bucket %v failed with error: %v", bucket, err)
		}
		for _, obj := range o.Results {
			t.Logf("Cleanup deletion of %v", obj.Name)
			if err = client.Bucket(bucket).Object(obj.Name).Delete(ctx); err != nil {
				t.Fatalf("Cleanup Delete for object %v failed with %v", obj.Name, err)
			}
		}
		if o.Next == nil {
			break
		}
		q = o.Next
	}
}
Ejemplo n.º 3
0
func Example_auth() {
	// Initialize an authorized context with Google Developers Console
	// JSON key. Read the google package examples to learn more about
	// different authorization flows you can use.
	// http://godoc.org/golang.org/x/oauth2/google
	jsonKey, err := ioutil.ReadFile("/path/to/json/keyfile.json")
	if err != nil {
		log.Fatal(err)
	}
	conf, err := google.JWTConfigFromJSON(
		jsonKey,
		storage.ScopeFullControl,
	)
	if err != nil {
		log.Fatal(err)
	}
	ctx := context.Background()
	client, err := storage.NewClient(ctx, cloud.WithTokenSource(conf.TokenSource(ctx)))
	if err != nil {
		log.Fatal(err)
	}

	// Use the client (see other examples)
	doSomething(client)

	// After using the client, free any resources (e.g. network connections).
	client.Close()
}
Ejemplo n.º 4
0
func main() {
	log.SetFlags(0)
	plugin.Param("workspace", &workspace)
	plugin.Param("build", &build)
	plugin.Param("repo", &repo)
	plugin.Param("vargs", &vargs)
	plugin.MustParse()
	sort.Strings(vargs.Gzip) // need for matchGzip

	// context for all clients
	ctx := context.Background()
	// GitHub client
	gts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: vargs.GitHubToken})
	client.ghub = github.NewClient(oauth2.NewClient(ctx, gts))
	// GCS client
	auth, err := google.JWTConfigFromJSON([]byte(vargs.AuthKey), storage.ScopeFullControl)
	if err != nil {
		fatalf("auth: %v", err)
	}
	tsrc := auth.TokenSource(ctx)
	client.gcs, err = storage.NewClient(ctx, cloud.WithTokenSource(auth.TokenSource(ctx)))
	if err != nil {
		fatalf("storage client: %v", err)
	}
	// http client with service account authorization
	client.http = oauth2.NewClient(ctx, tsrc)

	run()
	if ecode != 0 {
		msg := fmt.Sprintf("exited with code %d", ecode)
		updateStatus("error", msg, stagingURL)
	}
	os.Exit(ecode)
}
Ejemplo n.º 5
0
func TestAdminClient(t *testing.T) {
	ctx := context.Background()
	projectID := testutil.ProjID()
	newBucket := projectID + "copy"

	client, err := NewAdminClient(ctx, projectID, cloud.WithTokenSource(testutil.TokenSource(ctx, ScopeFullControl)))
	if err != nil {
		t.Fatalf("Could not create client: %v", err)
	}
	defer client.Close()

	if err := client.CreateBucket(ctx, newBucket, nil); err != nil {
		t.Errorf("CreateBucket(%v, %v) failed %v", newBucket, nil, err)
	}
	if err := client.DeleteBucket(ctx, newBucket); err != nil {
		t.Errorf("DeleteBucket(%v) failed %v", newBucket, err)
		t.Logf("TODO: Warning this test left a new bucket in the cloud project, it must be deleted manually")
	}
	attrs := BucketAttrs{
		DefaultObjectACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}},
	}
	if err := client.CreateBucket(ctx, newBucket, &attrs); err != nil {
		t.Errorf("CreateBucket(%v, %v) failed %v", newBucket, attrs, err)
	}
	if err := client.DeleteBucket(ctx, newBucket); err != nil {
		t.Errorf("DeleteBucket(%v) failed %v", newBucket, err)
		t.Logf("TODO: Warning this test left a new bucket in the cloud project, it must be deleted manually")
	}
}
Ejemplo n.º 6
0
func TestIntegrationPingBadProject(t *testing.T) {
	if testing.Short() {
		t.Skip("Integration tests skipped in short mode")
	}

	ctx := context.Background()
	ts := testutil.TokenSource(ctx, Scope)
	if ts == nil {
		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
	}

	for _, projID := range []string{
		testutil.ProjID() + "-BAD", // nonexistent project
		"amazing-height-519",       // exists, but wrong creds
	} {
		c, err := NewClient(ctx, projID, "logging-integration-test", cloud.WithTokenSource(ts))
		if err != nil {
			t.Fatalf("project %s: error creating client: %v", projID, err)
		}
		if err := c.Ping(); err == nil {
			t.Errorf("project %s: want error pinging logging api, got nil", projID)
		}
		// Ping twice, just to make sure the deduping doesn't mess with the result.
		if err := c.Ping(); err == nil {
			t.Errorf("project %s: want error pinging logging api, got nil", projID)
		}
	}
}
Ejemplo n.º 7
0
// TODO(djd): reevaluate this example given new Client config.
func Example_auth() *datastore.Client {
	// Initialize an authorized context with Google Developers Console
	// JSON key. Read the google package examples to learn more about
	// different authorization flows you can use.
	// http://godoc.org/golang.org/x/oauth2/google
	jsonKey, err := ioutil.ReadFile("/path/to/json/keyfile.json")
	if err != nil {
		log.Fatal(err)
	}
	conf, err := google.JWTConfigFromJSON(
		jsonKey,
		datastore.ScopeDatastore,
		datastore.ScopeUserEmail,
	)
	if err != nil {
		log.Fatal(err)
	}
	ctx := context.Background()
	client, err := datastore.NewClient(ctx, "project-id", cloud.WithTokenSource(conf.TokenSource(ctx)))
	if err != nil {
		log.Fatal(err)
	}
	// Use the client (see other examples).
	return client
}
Ejemplo n.º 8
0
func newClient(ctx context.Context) *Client {
	ts := testutil.TokenSource(ctx, ScopeDatastore, ScopeUserEmail)
	client, err := NewClient(ctx, testutil.ProjID(), cloud.WithTokenSource(ts))
	if err != nil {
		log.Fatal(err)
	}
	return client
}
Ejemplo n.º 9
0
func upload(srcPath string) {
	if !*flagUpload {
		return
	}
	destName := strings.Replace(filepath.Base(srcPath), "camlistore", "camlistore-"+rev(), 1)
	versionedTarball := "monthly/" + destName

	log.Printf("Uploading %s/%s ...", bucket, versionedTarball)

	ts, err := tokenSource(bucket)
	if err != nil {
		log.Fatal(err)
	}
	ctx := context.Background()
	stoClient, err := storage.NewClient(ctx, cloud.WithTokenSource(ts), cloud.WithBaseHTTP(oauth2.NewClient(ctx, ts)))
	if err != nil {
		log.Fatal(err)
	}
	w := stoClient.Bucket(bucket).Object(versionedTarball).NewWriter(ctx)
	w.ACL = publicACL(project)
	w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations?
	contentType := "application/x-gtar"
	if strings.HasSuffix(versionedTarball, ".zip") {
		contentType = "application/zip"
	}
	w.ContentType = contentType
	csw := sha256.New()
	mw := io.MultiWriter(w, csw)

	src, err := os.Open(srcPath)
	if err != nil {
		log.Fatal(err)
	}
	defer src.Close()

	if _, err := io.Copy(mw, src); err != nil {
		log.Fatalf("io.Copy: %v", err)
	}
	if err := w.Close(); err != nil {
		log.Fatalf("closing GCS storage writer: %v", err)
	}
	log.Printf("Uploaded monthly tarball to %s", versionedTarball)

	// And upload the corresponding checksum
	checkSumFile := versionedTarball + ".sha256"
	sum := fmt.Sprintf("%x", csw.Sum(nil))
	w = stoClient.Bucket(bucket).Object(checkSumFile).NewWriter(ctx)
	w.ACL = publicACL(project)
	w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations?
	w.ContentType = "text/plain"
	if _, err := io.Copy(w, strings.NewReader(sum)); err != nil {
		log.Fatalf("error uploading checksum %v: %v", checkSumFile, err)
	}
	if err := w.Close(); err != nil {
		log.Fatalf("closing GCS storage writer: %v", err)
	}
	log.Printf("Uploaded monthly tarball checksum to %s", checkSumFile)
}
func getAppDefaultClientArgs() (*context.Context, *cloud.ClientOption, error) {
	ctx := oauth2.NoContext
	tokenSrc, err := google.DefaultTokenSource(ctx, ScopeCloudPlatform)
	if err != nil {
		return nil, nil, err
	}

	clientOption := cloud.WithTokenSource(tokenSrc)
	return &ctx, &clientOption, nil
}
Ejemplo n.º 11
0
func (cred *ServerCred) AuthDatastore(ctx context.Context) (*datastore.Client, error) {
	ctx = TraceStart(ctx, "/frontend._.AuthDatastore")
	defer TraceEnd(ctx, cred)

	client, err := datastore.NewClient(
		ctx, ProjectId, cloud.WithTokenSource(cred.cred.TokenSource(ctx)))
	if err != nil {
		return nil, err
	}
	return client, nil
}
Ejemplo n.º 12
0
// uploadReleaseTarball uploads the generated tarball of binaries in
// camlistore-release/VERSION/camlistoreVERSION-REV-CONTENTS.EXT. It then makes a copy in
// the same bucket and path, as camlistoreVERSION-CONTENTS.EXT.
func uploadReleaseTarball() {
	proj := "camlistore-website"
	bucket := "camlistore-release"
	tarball := *flagVersion + "/" + filepath.Base(releaseTarball)
	versionedTarball := strings.Replace(tarball, "camlistore"+*flagVersion, "camlistore"+*flagVersion+"-"+rev(), 1)

	log.Printf("Uploading %s/%s ...", bucket, versionedTarball)

	ts, err := tokenSource(bucket)
	if err != nil {
		log.Fatal(err)
	}
	ctx := context.Background()
	stoClient, err := storage.NewClient(ctx, cloud.WithTokenSource(ts), cloud.WithBaseHTTP(oauth2.NewClient(ctx, ts)))
	if err != nil {
		log.Fatal(err)
	}
	w := stoClient.Bucket(bucket).Object(versionedTarball).NewWriter(ctx)
	w.ACL = publicACL(proj)
	w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations?
	contentType := "application/x-gtar"
	if *buildOS == "windows" {
		contentType = "application/zip"
	}
	w.ContentType = contentType

	src, err := os.Open(releaseTarball)
	if err != nil {
		log.Fatal(err)
	}
	defer src.Close()

	if _, err := io.Copy(w, src); err != nil {
		log.Fatalf("io.Copy: %v", err)
	}
	if err := w.Close(); err != nil {
		log.Fatalf("closing GCS storage writer: %v", err)
	}
	log.Printf("Uploaded tarball to %s", versionedTarball)
	if !isWIP() {
		log.Printf("Copying tarball to %s/%s ...", bucket, tarball)
		dest := stoClient.Bucket(bucket).Object(tarball)
		if _, err := stoClient.Bucket(bucket).Object(versionedTarball).CopyTo(
			ctx,
			dest,
			&storage.ObjectAttrs{
				ACL:         publicACL(proj),
				ContentType: contentType,
			}); err != nil {
			log.Fatalf("Error uploading %v: %v", tarball, err)
		}
		log.Printf("Uploaded tarball to %s", tarball)
	}
}
Ejemplo n.º 13
0
func newGoogleClient(ctx context.Context, bucket string) (*googleClient, error) {
	client, err := storage.NewClient(
		ctx,
		cloud.WithTokenSource(google.ComputeTokenSource("")),
		cloud.WithScopes(storage.ScopeFullControl),
	)
	if err != nil {
		return nil, err
	}
	return &googleClient{ctx, client.Bucket(bucket)}, nil
}
Ejemplo n.º 14
0
func newClient(ctx context.Context, t *testing.T) *Client {
	ts := testutil.TokenSource(ctx, ScopeDatastore)
	if ts == nil {
		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
	}
	client, err := NewClient(ctx, testutil.ProjID(), cloud.WithTokenSource(ts))
	if err != nil {
		t.Fatalf("NewClient: %v", err)
	}
	return client
}
func getServiceAccountClientArgs(jwtScope string) (*context.Context, *cloud.ClientOption, error) {
	jsonKey, err := ioutil.ReadFile(KeyFile)
	if err != nil {
		return nil, nil, err
	}

	config, err := google.JWTConfigFromJSON(jsonKey, jwtScope)
	if err != nil {
		return nil, nil, err
	}

	ctx := cloud.NewContext(ProjectID, config.Client(oauth2.NoContext))
	clientOption := cloud.WithTokenSource(config.TokenSource(ctx))
	return &ctx, &clientOption, nil
}
Ejemplo n.º 16
0
// config is like testConfig, but it doesn't need a *testing.T.
func config(ctx context.Context) (*Client, string) {
	ts := testutil.TokenSource(ctx, ScopeFullControl)
	if ts == nil {
		return nil, ""
	}
	p := testutil.ProjID()
	if p == "" {
		log.Fatal("The project ID must be set. See CONTRIBUTING.md for details")
	}
	client, err := NewClient(ctx, cloud.WithTokenSource(ts))
	if err != nil {
		log.Fatalf("NewClient: %v", err)
	}
	return client, p
}
Ejemplo n.º 17
0
// testConfig returns the Client used to access GCS and the default bucket
// name to use.
func testConfig(ctx context.Context, t *testing.T) (*Client, string) {
	ts := cloud.WithTokenSource(testutil.TokenSource(ctx, ScopeFullControl))
	if ts == nil {
		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
	}
	p := testutil.ProjID()
	if p == "" {
		log.Fatal("The project ID must be set. See CONTRIBUTING.md for details")
	}
	client, err := NewClient(ctx, ts)
	if err != nil {
		t.Fatalf("NewClient: %v", err)
	}
	return client, p
}
Ejemplo n.º 18
0
func TestValidObjectNames(t *testing.T) {
	ctx := context.Background()
	client, err := NewClient(ctx, cloud.WithTokenSource(testutil.TokenSource(ctx, ScopeFullControl)))
	if err != nil {
		t.Errorf("Could not create client: %v", err)
	}
	defer client.Close()

	bkt := client.Bucket(bucket)

	validNames := []string{
		"gopher",
		"Гоферови",
		"a",
		strings.Repeat("a", 1024),
	}
	for _, name := range validNames {
		w := bkt.Object(name).NewWriter(ctx)
		if _, err := w.Write([]byte("data")); err != nil {
			t.Errorf("Object %q write failed: %v. Want success", name, err)
			continue
		}
		if err := w.Close(); err != nil {
			t.Errorf("Object %q close failed: %v. Want success", name, err)
			continue
		}
		defer bkt.Object(name).Delete(ctx)
	}

	invalidNames := []string{
		"", // Too short.
		strings.Repeat("a", 1025), // Too long.
		"new\nlines",
		"bad\xffunicode",
	}
	for _, name := range invalidNames {
		w := bkt.Object(name).NewWriter(ctx)
		// Invalid object names will either cause failure during Write or Close.
		if _, err := w.Write([]byte("data")); err != nil {
			continue
		}
		if err := w.Close(); err != nil {
			continue
		}
		defer bkt.Object(name).Delete(ctx)
		t.Errorf("%q should have failed. Didn't", name)
	}
}
Ejemplo n.º 19
0
Archivo: sub.go Proyecto: Ropes/pubbing
// GCEClientInit uses Google's host FS searching functionality to find auth
// tokens if they exist. eg: GCE VMs, Authenticated Developers
func GCEClientInit(ctx *context.Context, project string) *pubsub.Client {
	var client *pubsub.Client
	clientOnce := new(sync.Once)
	clientOnce.Do(func() {
		source, err := google.DefaultTokenSource(*ctx, pubsub.ScopePubSub)
		if err != nil {
			log.Errorf("error creating token source: %v", err)
			os.Exit(1)
		}
		client, err = pubsub.NewClient(*ctx, project, cloud.WithTokenSource(source))
		if err != nil {
			log.Errorf("error creating pubsub.Client: %v", err)
			os.Exit(1)
		}
	})
	return client
}
Ejemplo n.º 20
0
func TestIntegration(t *testing.T) {
	if testing.Short() {
		t.Skip("Integration tests skipped in short mode")
	}

	ctx := context.Background()
	ts := testutil.TokenSource(ctx, Scope)
	if ts == nil {
		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
	}

	projID := testutil.ProjID()

	c, err := NewClient(ctx, projID, "logging-integration-test", cloud.WithTokenSource(ts))
	if err != nil {
		t.Fatalf("error creating client: %v", err)
	}

	if err := c.Ping(); err != nil {
		t.Fatalf("error pinging logging api: %v", err)
	}
	// Ping twice, to verify that deduping doesn't change the result.
	if err := c.Ping(); err != nil {
		t.Fatalf("error pinging logging api: %v", err)
	}

	if err := c.LogSync(Entry{Payload: customJSONObject{}}); err != nil {
		t.Fatalf("error writing log: %v", err)
	}

	if err := c.Log(Entry{Payload: customJSONObject{}}); err != nil {
		t.Fatalf("error writing log: %v", err)
	}

	if _, err := c.Writer(Default).Write([]byte("test log with io.Writer")); err != nil {
		t.Fatalf("error writing log using io.Writer: %v", err)
	}

	c.Logger(Default).Println("test log with log.Logger")

	if err := c.Flush(); err != nil {
		t.Fatalf("error flushing logs: %v", err)
	}
}
Ejemplo n.º 21
0
func main() {
	log.SetFlags(0)
	plugin.Param("workspace", &workspace)
	plugin.Param("vargs", &vargs)
	plugin.MustParse()
	sort.Strings(vargs.Gzip) // need for matchGzip

	auth, err := google.JWTConfigFromJSON([]byte(vargs.AuthKey), storage.ScopeFullControl)
	if err != nil {
		fatalf("auth: %v", err)
	}
	ctx := context.Background()
	client, err := storage.NewClient(ctx, cloud.WithTokenSource(auth.TokenSource(ctx)))
	if err != nil {
		fatalf("storage client: %v", err)
	}
	run(client)
	os.Exit(ecode)
}
Ejemplo n.º 22
0
Archivo: sub.go Proyecto: Ropes/pubbing
// JWTClientInit reads in a service account JSON token and creates an oauth
// token for communicating with GCE.
func JWTClientInit(ctx *context.Context) *pubsub.Client {
	jsonKey, err := ioutil.ReadFile(KeyPath)
	if err != nil {
		log.Errorf("error reading keyfile: %v", err)
		os.Exit(1)
	}

	conf, err := google.JWTConfigFromJSON(jsonKey, pubsub.ScopePubSub)
	if err != nil {
		log.Errorf("error creating conf file: %v", err)
	}

	oauthTokenSource := conf.TokenSource(*ctx)
	psClient, err := pubsub.NewClient(*ctx, Gceproject, cloud.WithTokenSource(oauthTokenSource))
	if err != nil {
		log.Errorf("error creating pubsub client: %v", err)
		os.Exit(1)
	}
	return psClient
}
func getClientOptionFromJsonKeyFile(ctx context.Context, keyFilePath string, scope string) cloud.ClientOption {

	fmt.Println("Reading [", keyFilePath, "] ...")
	jsonKey, err := ioutil.ReadFile(keyFilePath)

	if err != nil {
		fmt.Println("Error on [ioutil.ReadFile]: %v", err)
	}

	fmt.Println("Creating config ...")
	config, err := google.JWTConfigFromJSON(jsonKey, scope)

	if err != nil {
		fmt.Println("Error on [google.JWTConfigFromJSON]]: %v", err)
	}

	clientOption := cloud.WithTokenSource(config.TokenSource(ctx))

	return clientOption
}
Ejemplo n.º 24
0
func newLogTest(t *testing.T) *logTest {
	handlerc := make(chan http.Handler, 1)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		select {
		case h := <-handlerc:
			h.ServeHTTP(w, r)
		default:
			slurp, _ := ioutil.ReadAll(r.Body)
			t.Errorf("Unexpected HTTP request received: %s", slurp)
			w.WriteHeader(500)
			io.WriteString(w, "unexpected HTTP request")
		}
	}))
	c, err := NewClient(context.Background(), "PROJ-ID", "LOG-NAME",
		cloud.WithEndpoint(ts.URL),
		cloud.WithTokenSource(dummyTokenSource{}), // prevent DefaultTokenSource
	)
	if err != nil {
		t.Fatal(err)
	}
	var clock struct {
		sync.Mutex
		now time.Time
	}
	c.timeNow = func() time.Time {
		clock.Lock()
		defer clock.Unlock()
		if clock.now.IsZero() {
			clock.now = time.Unix(0, 0)
		}
		clock.now = clock.now.Add(1 * time.Second)
		return clock.now
	}
	return &logTest{
		t:        t,
		ts:       ts,
		c:        c,
		handlerc: handlerc,
	}
}
Ejemplo n.º 25
0
func main() {
	fmt.Printf("Drone Google Cloud Storage Plugin built from %s\n", buildCommit)

	log.SetFlags(0)
	plugin.Param("workspace", &workspace)
	plugin.Param("vargs", &vargs)
	plugin.MustParse()
	sort.Strings(vargs.Gzip) // need for matchGzip
	rand.Seed(time.Now().UnixNano())

	auth, err := google.JWTConfigFromJSON([]byte(vargs.AuthKey), storage.ScopeFullControl)
	if err != nil {
		fatalf("auth: %v", err)
	}
	ctx := context.Background()
	client, err := storage.NewClient(ctx, cloud.WithTokenSource(auth.TokenSource(ctx)))
	if err != nil {
		fatalf("storage client: %v", err)
	}
	run(client)
	os.Exit(ecode)
}
Ejemplo n.º 26
0
func TestAdminClient(t *testing.T) {
	if testing.Short() {
		t.Skip("Integration tests skipped in short mode")
	}
	ctx := context.Background()
	ts := testutil.TokenSource(ctx, ScopeFullControl)
	if ts == nil {
		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
	}
	projectID := testutil.ProjID()

	newBucket := projectID + suffix
	t.Logf("Testing admin with Bucket %q", newBucket)

	client, err := NewAdminClient(ctx, projectID, cloud.WithTokenSource(ts))
	if err != nil {
		t.Fatalf("Could not create client: %v", err)
	}
	defer client.Close()

	if err := client.CreateBucket(ctx, newBucket, nil); err != nil {
		t.Errorf("CreateBucket(%v, %v) failed %v", newBucket, nil, err)
	}
	if err := client.DeleteBucket(ctx, newBucket); err != nil {
		t.Errorf("DeleteBucket(%v) failed %v", newBucket, err)
		t.Logf("TODO: Warning this test left a new bucket in the cloud project, it must be deleted manually")
	}
	attrs := BucketAttrs{
		DefaultObjectACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}},
	}
	if err := client.CreateBucket(ctx, newBucket, &attrs); err != nil {
		t.Errorf("CreateBucket(%v, %v) failed %v", newBucket, attrs, err)
	}
	if err := client.DeleteBucket(ctx, newBucket); err != nil {
		t.Errorf("DeleteBucket(%v) failed %v", newBucket, err)
		t.Logf("TODO: Warning this test left a new bucket in the cloud project, it must be deleted manually")
	}
}
Ejemplo n.º 27
0
func gcloudAuth() {
	jsonKey, err := ioutil.ReadFile(config.CredentialsPath)
	if err != nil {
		log.Fatal(err)
	}
	conf, err := google.JWTConfigFromJSON(
		jsonKey,
		pubsub.ScopeCloudPlatform,
		pubsub.ScopePubSub,
		storage.ScopeFullControl,
	)
	if err != nil {
		log.Fatal(err)
	}

	ctx := cloud.NewContext(config.GsProjectID, conf.Client(oauth2.NoContext))
	client, err := storage.NewClient(ctx, cloud.WithTokenSource(conf.TokenSource(ctx)))
	if err != nil {
		log.Fatal("New Storage client.", err)
	}

	gcloud.ctx = ctx
	gcloud.storageClient = *client
}
Ejemplo n.º 28
0
func (cfg *Config) getClient(ctx context.Context) (*datastore.Client, error) {
	return datastore.NewClient(ctx, cfg.projectName, cloud.WithTokenSource(cfg.tsrc))
}
Ejemplo n.º 29
0
func TestAll(t *testing.T) {
	if testing.Short() {
		t.Skip("Integration tests skipped in short mode")
	}
	// TODO(djd): Replace this ctx with context.Background() when the new API is complete.
	ctx := testutil.Context(ScopePubSub, ScopeCloudPlatform)
	if ctx == nil {
		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
	}
	ts := testutil.TokenSource(ctx, ScopePubSub, ScopeCloudPlatform)
	if ts == nil {
		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
	}

	now := time.Now()
	topicName := fmt.Sprintf("topic-%d", now.Unix())
	subName := fmt.Sprintf("subscription-%d", now.Unix())

	client, err := NewClient(ctx, testutil.ProjID(), cloud.WithTokenSource(ts))
	if err != nil {
		t.Fatalf("Creating client error: %v", err)
	}

	var topic *TopicHandle
	if topic, err = client.NewTopic(ctx, topicName); err != nil {
		t.Errorf("CreateTopic error: %v", err)
	}

	var sub *SubscriptionHandle
	if sub, err = topic.Subscribe(ctx, subName, nil); err != nil {
		t.Errorf("CreateSub error: %v", err)
	}

	exists, err := topic.Exists(ctx)
	if err != nil {
		t.Fatalf("TopicExists error: %v", err)
	}
	if !exists {
		t.Errorf("topic %s should exist, but it doesn't", topic)
	}

	exists, err = sub.Exists(ctx)
	if err != nil {
		t.Fatalf("SubExists error: %v", err)
	}
	if !exists {
		t.Errorf("subscription %s should exist, but it doesn't", subName)
	}

	max := 10
	msgs := make([]*Message, max)
	expectedMsgs := make(map[string]bool, max)
	for i := 0; i < max; i++ {
		text := fmt.Sprintf("a message with an index %d", i)
		attrs := make(map[string]string)
		attrs["foo"] = "bar"
		msgs[i] = &Message{
			Data:       []byte(text),
			Attributes: attrs,
		}
		expectedMsgs[text] = false
	}

	ids, err := Publish(ctx, topicName, msgs...)
	if err != nil {
		t.Fatalf("Publish (1) error: %v", err)
	}

	if len(ids) != max {
		t.Errorf("unexpected number of message IDs received; %d, want %d", len(ids), max)
	}

	expectedIDs := make(map[string]bool, max)
	for _, id := range ids {
		expectedIDs[id] = false
	}

	received, err := PullWait(ctx, subName, max)
	if err != nil {
		t.Fatalf("PullWait error: %v", err)
	}

	if len(received) != max {
		t.Errorf("unexpected number of messages received; %d, want %d", len(received), max)
	}

	for _, msg := range received {
		expectedMsgs[string(msg.Data)] = true
		expectedIDs[msg.ID] = true
		if msg.Attributes["foo"] != "bar" {
			t.Errorf("message attribute foo is expected to be 'bar', found '%s'", msg.Attributes["foo"])
		}
	}

	for msg, found := range expectedMsgs {
		if !found {
			t.Errorf("message '%s' should be received", msg)
		}
	}

	for id, found := range expectedIDs {
		if !found {
			t.Errorf("message with the message id '%s' should be received", id)
		}
	}

	// base64 test
	data := "=@~"
	msg := &Message{
		Data: []byte(data),
	}
	_, err = Publish(ctx, topicName, msg)
	if err != nil {
		t.Fatalf("Publish (2) error: %v", err)
	}

	received, err = PullWait(ctx, subName, 1)
	if err != nil {
		t.Fatalf("PullWait error: %v", err)
	}
	if len(received) != 1 {
		t.Fatalf("unexpected number of messages received; %d, want %d", len(received), 1)
	}
	if string(received[0].Data) != data {
		t.Errorf("unexpexted message received; %s, want %s", string(received[0].Data), data)
	}

	err = sub.Delete(ctx)
	if err != nil {
		t.Errorf("DeleteSub error: %v", err)
	}

	err = topic.Delete(ctx)
	if err != nil {
		t.Errorf("DeleteTopic error: %v", err)
	}
}
Ejemplo n.º 30
0
// uploadDockerImage makes a tar.gz snapshot of the camlistored docker image,
// and uploads it at camlistore-release/docker/camlistored-REV.tar.gz. It then
// makes a copy in the same bucket and path as camlistored.tar.gz.
func uploadDockerImage() {
	proj := "camlistore-website"
	bucket := "camlistore-release"
	versionedTarball := "docker/camlistored-" + rev() + ".tar.gz"
	tarball := "docker/camlistored.tar.gz"

	log.Printf("Uploading %s/%s ...", bucket, versionedTarball)

	ts, err := tokenSource(bucket)
	if err != nil {
		log.Fatal(err)
	}
	ctx := context.Background()
	stoClient, err := storage.NewClient(ctx, cloud.WithTokenSource(ts), cloud.WithBaseHTTP(oauth2.NewClient(ctx, ts)))
	if err != nil {
		log.Fatal(err)
	}
	w := stoClient.Bucket(bucket).Object(versionedTarball).NewWriter(ctx)
	w.ACL = publicACL(proj)
	w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations?
	w.ContentType = "application/x-gtar"

	dockerSave := exec.Command("docker", "save", serverImage)
	dockerSave.Stderr = os.Stderr
	tar, err := dockerSave.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	targz, pw := io.Pipe()
	go func() {
		zw := gzip.NewWriter(pw)
		n, err := io.Copy(zw, tar)
		if err != nil {
			log.Fatalf("Error copying to gzip writer: after %d bytes, %v", n, err)
		}
		if err := zw.Close(); err != nil {
			log.Fatalf("gzip.Close: %v", err)
		}
		pw.CloseWithError(err)
	}()
	if err := dockerSave.Start(); err != nil {
		log.Fatalf("Error starting docker save %v: %v", serverImage, err)
	}
	if _, err := io.Copy(w, targz); err != nil {
		log.Fatalf("io.Copy: %v", err)
	}
	if err := w.Close(); err != nil {
		log.Fatalf("closing GCS storage writer: %v", err)
	}
	if err := dockerSave.Wait(); err != nil {
		log.Fatalf("Error waiting for docker save %v: %v", serverImage, err)
	}
	log.Printf("Uploaded tarball to %s", versionedTarball)
	if !isWIP() {
		log.Printf("Copying tarball to %s/%s ...", bucket, tarball)
		if _, err := stoClient.CopyObject(ctx,
			bucket, versionedTarball,
			bucket, tarball,
			&storage.ObjectAttrs{
				ACL:          publicACL(proj),
				CacheControl: "no-cache",
				ContentType:  "application/x-gtar",
			}); err != nil {
			log.Fatalf("Error uploading %v: %v", tarball, err)
		}
		log.Printf("Uploaded tarball to %s", tarball)
	}
}