func GetNewStorageClient() (*StorageClient, error) {
client, err := storage.NewClient(ctx, cloud.WithTokenSource(defaultTokenSource))
return &StorageClient{
Client: client,
Ctx: ctx,
}, err
}
func cleanup(t *testing.T, prefix string) {
ctx := context.Background()
client, err := NewClient(ctx, cloud.WithTokenSource(testutil.TokenSource(ctx, ScopeFullControl)))
if err != nil {
t.Errorf("Could not create client: %v", err)
}
defer client.Close()
var q *Query = &Query{
Prefix: prefix,
}
for {
o, err := client.Bucket(bucket).List(ctx, q)
if err != nil {
t.Fatalf("Cleanup List for bucket %v failed with error: %v", bucket, err)
}
for _, obj := range o.Results {
t.Logf("Cleanup deletion of %v", obj.Name)
if err = client.Bucket(bucket).Object(obj.Name).Delete(ctx); err != nil {
t.Fatalf("Cleanup Delete for object %v failed with %v", obj.Name, err)
}
}
if o.Next == nil {
break
}
q = o.Next
}
}
func Example_auth() {
// Initialize an authorized context with Google Developers Console
// JSON key. Read the google package examples to learn more about
// different authorization flows you can use.
// http://godoc.org/golang.org/x/oauth2/google
jsonKey, err := ioutil.ReadFile("/path/to/json/keyfile.json")
if err != nil {
log.Fatal(err)
}
conf, err := google.JWTConfigFromJSON(
jsonKey,
storage.ScopeFullControl,
)
if err != nil {
log.Fatal(err)
}
ctx := context.Background()
client, err := storage.NewClient(ctx, cloud.WithTokenSource(conf.TokenSource(ctx)))
if err != nil {
log.Fatal(err)
}
// Use the client (see other examples)
doSomething(client)
// After using the client, free any resources (e.g. network connections).
client.Close()
}
func main() {
log.SetFlags(0)
plugin.Param("workspace", &workspace)
plugin.Param("build", &build)
plugin.Param("repo", &repo)
plugin.Param("vargs", &vargs)
plugin.MustParse()
sort.Strings(vargs.Gzip) // need for matchGzip
// context for all clients
ctx := context.Background()
// GitHub client
gts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: vargs.GitHubToken})
client.ghub = github.NewClient(oauth2.NewClient(ctx, gts))
// GCS client
auth, err := google.JWTConfigFromJSON([]byte(vargs.AuthKey), storage.ScopeFullControl)
if err != nil {
fatalf("auth: %v", err)
}
tsrc := auth.TokenSource(ctx)
client.gcs, err = storage.NewClient(ctx, cloud.WithTokenSource(auth.TokenSource(ctx)))
if err != nil {
fatalf("storage client: %v", err)
}
// http client with service account authorization
client.http = oauth2.NewClient(ctx, tsrc)
run()
if ecode != 0 {
msg := fmt.Sprintf("exited with code %d", ecode)
updateStatus("error", msg, stagingURL)
}
os.Exit(ecode)
}
func TestAdminClient(t *testing.T) {
ctx := context.Background()
projectID := testutil.ProjID()
newBucket := projectID + "copy"
client, err := NewAdminClient(ctx, projectID, cloud.WithTokenSource(testutil.TokenSource(ctx, ScopeFullControl)))
if err != nil {
t.Fatalf("Could not create client: %v", err)
}
defer client.Close()
if err := client.CreateBucket(ctx, newBucket, nil); err != nil {
t.Errorf("CreateBucket(%v, %v) failed %v", newBucket, nil, err)
}
if err := client.DeleteBucket(ctx, newBucket); err != nil {
t.Errorf("DeleteBucket(%v) failed %v", newBucket, err)
t.Logf("TODO: Warning this test left a new bucket in the cloud project, it must be deleted manually")
}
attrs := BucketAttrs{
DefaultObjectACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}},
}
if err := client.CreateBucket(ctx, newBucket, &attrs); err != nil {
t.Errorf("CreateBucket(%v, %v) failed %v", newBucket, attrs, err)
}
if err := client.DeleteBucket(ctx, newBucket); err != nil {
t.Errorf("DeleteBucket(%v) failed %v", newBucket, err)
t.Logf("TODO: Warning this test left a new bucket in the cloud project, it must be deleted manually")
}
}
func TestIntegrationPingBadProject(t *testing.T) {
if testing.Short() {
t.Skip("Integration tests skipped in short mode")
}
ctx := context.Background()
ts := testutil.TokenSource(ctx, Scope)
if ts == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
for _, projID := range []string{
testutil.ProjID() + "-BAD", // nonexistent project
"amazing-height-519", // exists, but wrong creds
} {
c, err := NewClient(ctx, projID, "logging-integration-test", cloud.WithTokenSource(ts))
if err != nil {
t.Fatalf("project %s: error creating client: %v", projID, err)
}
if err := c.Ping(); err == nil {
t.Errorf("project %s: want error pinging logging api, got nil", projID)
}
// Ping twice, just to make sure the deduping doesn't mess with the result.
if err := c.Ping(); err == nil {
t.Errorf("project %s: want error pinging logging api, got nil", projID)
}
}
}
// TODO(djd): reevaluate this example given new Client config.
func Example_auth() *datastore.Client {
// Initialize an authorized context with Google Developers Console
// JSON key. Read the google package examples to learn more about
// different authorization flows you can use.
// http://godoc.org/golang.org/x/oauth2/google
jsonKey, err := ioutil.ReadFile("/path/to/json/keyfile.json")
if err != nil {
log.Fatal(err)
}
conf, err := google.JWTConfigFromJSON(
jsonKey,
datastore.ScopeDatastore,
datastore.ScopeUserEmail,
)
if err != nil {
log.Fatal(err)
}
ctx := context.Background()
client, err := datastore.NewClient(ctx, "project-id", cloud.WithTokenSource(conf.TokenSource(ctx)))
if err != nil {
log.Fatal(err)
}
// Use the client (see other examples).
return client
}
func newClient(ctx context.Context) *Client {
ts := testutil.TokenSource(ctx, ScopeDatastore, ScopeUserEmail)
client, err := NewClient(ctx, testutil.ProjID(), cloud.WithTokenSource(ts))
if err != nil {
log.Fatal(err)
}
return client
}
func upload(srcPath string) {
if !*flagUpload {
return
}
destName := strings.Replace(filepath.Base(srcPath), "camlistore", "camlistore-"+rev(), 1)
versionedTarball := "monthly/" + destName
log.Printf("Uploading %s/%s ...", bucket, versionedTarball)
ts, err := tokenSource(bucket)
if err != nil {
log.Fatal(err)
}
ctx := context.Background()
stoClient, err := storage.NewClient(ctx, cloud.WithTokenSource(ts), cloud.WithBaseHTTP(oauth2.NewClient(ctx, ts)))
if err != nil {
log.Fatal(err)
}
w := stoClient.Bucket(bucket).Object(versionedTarball).NewWriter(ctx)
w.ACL = publicACL(project)
w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations?
contentType := "application/x-gtar"
if strings.HasSuffix(versionedTarball, ".zip") {
contentType = "application/zip"
}
w.ContentType = contentType
csw := sha256.New()
mw := io.MultiWriter(w, csw)
src, err := os.Open(srcPath)
if err != nil {
log.Fatal(err)
}
defer src.Close()
if _, err := io.Copy(mw, src); err != nil {
log.Fatalf("io.Copy: %v", err)
}
if err := w.Close(); err != nil {
log.Fatalf("closing GCS storage writer: %v", err)
}
log.Printf("Uploaded monthly tarball to %s", versionedTarball)
// And upload the corresponding checksum
checkSumFile := versionedTarball + ".sha256"
sum := fmt.Sprintf("%x", csw.Sum(nil))
w = stoClient.Bucket(bucket).Object(checkSumFile).NewWriter(ctx)
w.ACL = publicACL(project)
w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations?
w.ContentType = "text/plain"
if _, err := io.Copy(w, strings.NewReader(sum)); err != nil {
log.Fatalf("error uploading checksum %v: %v", checkSumFile, err)
}
if err := w.Close(); err != nil {
log.Fatalf("closing GCS storage writer: %v", err)
}
log.Printf("Uploaded monthly tarball checksum to %s", checkSumFile)
}
func getAppDefaultClientArgs() (*context.Context, *cloud.ClientOption, error) {
ctx := oauth2.NoContext
tokenSrc, err := google.DefaultTokenSource(ctx, ScopeCloudPlatform)
if err != nil {
return nil, nil, err
}
clientOption := cloud.WithTokenSource(tokenSrc)
return &ctx, &clientOption, nil
}
func (cred *ServerCred) AuthDatastore(ctx context.Context) (*datastore.Client, error) {
ctx = TraceStart(ctx, "/frontend._.AuthDatastore")
defer TraceEnd(ctx, cred)
client, err := datastore.NewClient(
ctx, ProjectId, cloud.WithTokenSource(cred.cred.TokenSource(ctx)))
if err != nil {
return nil, err
}
return client, nil
}
// uploadReleaseTarball uploads the generated tarball of binaries in
// camlistore-release/VERSION/camlistoreVERSION-REV-CONTENTS.EXT. It then makes a copy in
// the same bucket and path, as camlistoreVERSION-CONTENTS.EXT.
func uploadReleaseTarball() {
proj := "camlistore-website"
bucket := "camlistore-release"
tarball := *flagVersion + "/" + filepath.Base(releaseTarball)
versionedTarball := strings.Replace(tarball, "camlistore"+*flagVersion, "camlistore"+*flagVersion+"-"+rev(), 1)
log.Printf("Uploading %s/%s ...", bucket, versionedTarball)
ts, err := tokenSource(bucket)
if err != nil {
log.Fatal(err)
}
ctx := context.Background()
stoClient, err := storage.NewClient(ctx, cloud.WithTokenSource(ts), cloud.WithBaseHTTP(oauth2.NewClient(ctx, ts)))
if err != nil {
log.Fatal(err)
}
w := stoClient.Bucket(bucket).Object(versionedTarball).NewWriter(ctx)
w.ACL = publicACL(proj)
w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations?
contentType := "application/x-gtar"
if *buildOS == "windows" {
contentType = "application/zip"
}
w.ContentType = contentType
src, err := os.Open(releaseTarball)
if err != nil {
log.Fatal(err)
}
defer src.Close()
if _, err := io.Copy(w, src); err != nil {
log.Fatalf("io.Copy: %v", err)
}
if err := w.Close(); err != nil {
log.Fatalf("closing GCS storage writer: %v", err)
}
log.Printf("Uploaded tarball to %s", versionedTarball)
if !isWIP() {
log.Printf("Copying tarball to %s/%s ...", bucket, tarball)
dest := stoClient.Bucket(bucket).Object(tarball)
if _, err := stoClient.Bucket(bucket).Object(versionedTarball).CopyTo(
ctx,
dest,
&storage.ObjectAttrs{
ACL: publicACL(proj),
ContentType: contentType,
}); err != nil {
log.Fatalf("Error uploading %v: %v", tarball, err)
}
log.Printf("Uploaded tarball to %s", tarball)
}
}
func newGoogleClient(ctx context.Context, bucket string) (*googleClient, error) {
client, err := storage.NewClient(
ctx,
cloud.WithTokenSource(google.ComputeTokenSource("")),
cloud.WithScopes(storage.ScopeFullControl),
)
if err != nil {
return nil, err
}
return &googleClient{ctx, client.Bucket(bucket)}, nil
}
func newClient(ctx context.Context, t *testing.T) *Client {
ts := testutil.TokenSource(ctx, ScopeDatastore)
if ts == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
client, err := NewClient(ctx, testutil.ProjID(), cloud.WithTokenSource(ts))
if err != nil {
t.Fatalf("NewClient: %v", err)
}
return client
}
func getServiceAccountClientArgs(jwtScope string) (*context.Context, *cloud.ClientOption, error) {
jsonKey, err := ioutil.ReadFile(KeyFile)
if err != nil {
return nil, nil, err
}
config, err := google.JWTConfigFromJSON(jsonKey, jwtScope)
if err != nil {
return nil, nil, err
}
ctx := cloud.NewContext(ProjectID, config.Client(oauth2.NoContext))
clientOption := cloud.WithTokenSource(config.TokenSource(ctx))
return &ctx, &clientOption, nil
}
// config is like testConfig, but it doesn't need a *testing.T.
func config(ctx context.Context) (*Client, string) {
ts := testutil.TokenSource(ctx, ScopeFullControl)
if ts == nil {
return nil, ""
}
p := testutil.ProjID()
if p == "" {
log.Fatal("The project ID must be set. See CONTRIBUTING.md for details")
}
client, err := NewClient(ctx, cloud.WithTokenSource(ts))
if err != nil {
log.Fatalf("NewClient: %v", err)
}
return client, p
}
// testConfig returns the Client used to access GCS and the default bucket
// name to use.
func testConfig(ctx context.Context, t *testing.T) (*Client, string) {
ts := cloud.WithTokenSource(testutil.TokenSource(ctx, ScopeFullControl))
if ts == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
p := testutil.ProjID()
if p == "" {
log.Fatal("The project ID must be set. See CONTRIBUTING.md for details")
}
client, err := NewClient(ctx, ts)
if err != nil {
t.Fatalf("NewClient: %v", err)
}
return client, p
}
func TestValidObjectNames(t *testing.T) {
ctx := context.Background()
client, err := NewClient(ctx, cloud.WithTokenSource(testutil.TokenSource(ctx, ScopeFullControl)))
if err != nil {
t.Errorf("Could not create client: %v", err)
}
defer client.Close()
bkt := client.Bucket(bucket)
validNames := []string{
"gopher",
"Гоферови",
"a",
strings.Repeat("a", 1024),
}
for _, name := range validNames {
w := bkt.Object(name).NewWriter(ctx)
if _, err := w.Write([]byte("data")); err != nil {
t.Errorf("Object %q write failed: %v. Want success", name, err)
continue
}
if err := w.Close(); err != nil {
t.Errorf("Object %q close failed: %v. Want success", name, err)
continue
}
defer bkt.Object(name).Delete(ctx)
}
invalidNames := []string{
"", // Too short.
strings.Repeat("a", 1025), // Too long.
"new\nlines",
"bad\xffunicode",
}
for _, name := range invalidNames {
w := bkt.Object(name).NewWriter(ctx)
// Invalid object names will either cause failure during Write or Close.
if _, err := w.Write([]byte("data")); err != nil {
continue
}
if err := w.Close(); err != nil {
continue
}
defer bkt.Object(name).Delete(ctx)
t.Errorf("%q should have failed. Didn't", name)
}
}
// GCEClientInit uses Google's host FS searching functionality to find auth
// tokens if they exist. eg: GCE VMs, Authenticated Developers
func GCEClientInit(ctx *context.Context, project string) *pubsub.Client {
var client *pubsub.Client
clientOnce := new(sync.Once)
clientOnce.Do(func() {
source, err := google.DefaultTokenSource(*ctx, pubsub.ScopePubSub)
if err != nil {
log.Errorf("error creating token source: %v", err)
os.Exit(1)
}
client, err = pubsub.NewClient(*ctx, project, cloud.WithTokenSource(source))
if err != nil {
log.Errorf("error creating pubsub.Client: %v", err)
os.Exit(1)
}
})
return client
}
func TestIntegration(t *testing.T) {
if testing.Short() {
t.Skip("Integration tests skipped in short mode")
}
ctx := context.Background()
ts := testutil.TokenSource(ctx, Scope)
if ts == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
projID := testutil.ProjID()
c, err := NewClient(ctx, projID, "logging-integration-test", cloud.WithTokenSource(ts))
if err != nil {
t.Fatalf("error creating client: %v", err)
}
if err := c.Ping(); err != nil {
t.Fatalf("error pinging logging api: %v", err)
}
// Ping twice, to verify that deduping doesn't change the result.
if err := c.Ping(); err != nil {
t.Fatalf("error pinging logging api: %v", err)
}
if err := c.LogSync(Entry{Payload: customJSONObject{}}); err != nil {
t.Fatalf("error writing log: %v", err)
}
if err := c.Log(Entry{Payload: customJSONObject{}}); err != nil {
t.Fatalf("error writing log: %v", err)
}
if _, err := c.Writer(Default).Write([]byte("test log with io.Writer")); err != nil {
t.Fatalf("error writing log using io.Writer: %v", err)
}
c.Logger(Default).Println("test log with log.Logger")
if err := c.Flush(); err != nil {
t.Fatalf("error flushing logs: %v", err)
}
}
func main() {
log.SetFlags(0)
plugin.Param("workspace", &workspace)
plugin.Param("vargs", &vargs)
plugin.MustParse()
sort.Strings(vargs.Gzip) // need for matchGzip
auth, err := google.JWTConfigFromJSON([]byte(vargs.AuthKey), storage.ScopeFullControl)
if err != nil {
fatalf("auth: %v", err)
}
ctx := context.Background()
client, err := storage.NewClient(ctx, cloud.WithTokenSource(auth.TokenSource(ctx)))
if err != nil {
fatalf("storage client: %v", err)
}
run(client)
os.Exit(ecode)
}
// JWTClientInit reads in a service account JSON token and creates an oauth
// token for communicating with GCE.
func JWTClientInit(ctx *context.Context) *pubsub.Client {
jsonKey, err := ioutil.ReadFile(KeyPath)
if err != nil {
log.Errorf("error reading keyfile: %v", err)
os.Exit(1)
}
conf, err := google.JWTConfigFromJSON(jsonKey, pubsub.ScopePubSub)
if err != nil {
log.Errorf("error creating conf file: %v", err)
}
oauthTokenSource := conf.TokenSource(*ctx)
psClient, err := pubsub.NewClient(*ctx, Gceproject, cloud.WithTokenSource(oauthTokenSource))
if err != nil {
log.Errorf("error creating pubsub client: %v", err)
os.Exit(1)
}
return psClient
}
func getClientOptionFromJsonKeyFile(ctx context.Context, keyFilePath string, scope string) cloud.ClientOption {
fmt.Println("Reading [", keyFilePath, "] ...")
jsonKey, err := ioutil.ReadFile(keyFilePath)
if err != nil {
fmt.Println("Error on [ioutil.ReadFile]: %v", err)
}
fmt.Println("Creating config ...")
config, err := google.JWTConfigFromJSON(jsonKey, scope)
if err != nil {
fmt.Println("Error on [google.JWTConfigFromJSON]]: %v", err)
}
clientOption := cloud.WithTokenSource(config.TokenSource(ctx))
return clientOption
}
func newLogTest(t *testing.T) *logTest {
handlerc := make(chan http.Handler, 1)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
select {
case h := <-handlerc:
h.ServeHTTP(w, r)
default:
slurp, _ := ioutil.ReadAll(r.Body)
t.Errorf("Unexpected HTTP request received: %s", slurp)
w.WriteHeader(500)
io.WriteString(w, "unexpected HTTP request")
}
}))
c, err := NewClient(context.Background(), "PROJ-ID", "LOG-NAME",
cloud.WithEndpoint(ts.URL),
cloud.WithTokenSource(dummyTokenSource{}), // prevent DefaultTokenSource
)
if err != nil {
t.Fatal(err)
}
var clock struct {
sync.Mutex
now time.Time
}
c.timeNow = func() time.Time {
clock.Lock()
defer clock.Unlock()
if clock.now.IsZero() {
clock.now = time.Unix(0, 0)
}
clock.now = clock.now.Add(1 * time.Second)
return clock.now
}
return &logTest{
t: t,
ts: ts,
c: c,
handlerc: handlerc,
}
}
func main() {
fmt.Printf("Drone Google Cloud Storage Plugin built from %s\n", buildCommit)
log.SetFlags(0)
plugin.Param("workspace", &workspace)
plugin.Param("vargs", &vargs)
plugin.MustParse()
sort.Strings(vargs.Gzip) // need for matchGzip
rand.Seed(time.Now().UnixNano())
auth, err := google.JWTConfigFromJSON([]byte(vargs.AuthKey), storage.ScopeFullControl)
if err != nil {
fatalf("auth: %v", err)
}
ctx := context.Background()
client, err := storage.NewClient(ctx, cloud.WithTokenSource(auth.TokenSource(ctx)))
if err != nil {
fatalf("storage client: %v", err)
}
run(client)
os.Exit(ecode)
}
func TestAdminClient(t *testing.T) {
if testing.Short() {
t.Skip("Integration tests skipped in short mode")
}
ctx := context.Background()
ts := testutil.TokenSource(ctx, ScopeFullControl)
if ts == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
projectID := testutil.ProjID()
newBucket := projectID + suffix
t.Logf("Testing admin with Bucket %q", newBucket)
client, err := NewAdminClient(ctx, projectID, cloud.WithTokenSource(ts))
if err != nil {
t.Fatalf("Could not create client: %v", err)
}
defer client.Close()
if err := client.CreateBucket(ctx, newBucket, nil); err != nil {
t.Errorf("CreateBucket(%v, %v) failed %v", newBucket, nil, err)
}
if err := client.DeleteBucket(ctx, newBucket); err != nil {
t.Errorf("DeleteBucket(%v) failed %v", newBucket, err)
t.Logf("TODO: Warning this test left a new bucket in the cloud project, it must be deleted manually")
}
attrs := BucketAttrs{
DefaultObjectACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}},
}
if err := client.CreateBucket(ctx, newBucket, &attrs); err != nil {
t.Errorf("CreateBucket(%v, %v) failed %v", newBucket, attrs, err)
}
if err := client.DeleteBucket(ctx, newBucket); err != nil {
t.Errorf("DeleteBucket(%v) failed %v", newBucket, err)
t.Logf("TODO: Warning this test left a new bucket in the cloud project, it must be deleted manually")
}
}
func gcloudAuth() {
jsonKey, err := ioutil.ReadFile(config.CredentialsPath)
if err != nil {
log.Fatal(err)
}
conf, err := google.JWTConfigFromJSON(
jsonKey,
pubsub.ScopeCloudPlatform,
pubsub.ScopePubSub,
storage.ScopeFullControl,
)
if err != nil {
log.Fatal(err)
}
ctx := cloud.NewContext(config.GsProjectID, conf.Client(oauth2.NoContext))
client, err := storage.NewClient(ctx, cloud.WithTokenSource(conf.TokenSource(ctx)))
if err != nil {
log.Fatal("New Storage client.", err)
}
gcloud.ctx = ctx
gcloud.storageClient = *client
}
func (cfg *Config) getClient(ctx context.Context) (*datastore.Client, error) {
return datastore.NewClient(ctx, cfg.projectName, cloud.WithTokenSource(cfg.tsrc))
}
func TestAll(t *testing.T) {
if testing.Short() {
t.Skip("Integration tests skipped in short mode")
}
// TODO(djd): Replace this ctx with context.Background() when the new API is complete.
ctx := testutil.Context(ScopePubSub, ScopeCloudPlatform)
if ctx == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
ts := testutil.TokenSource(ctx, ScopePubSub, ScopeCloudPlatform)
if ts == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
now := time.Now()
topicName := fmt.Sprintf("topic-%d", now.Unix())
subName := fmt.Sprintf("subscription-%d", now.Unix())
client, err := NewClient(ctx, testutil.ProjID(), cloud.WithTokenSource(ts))
if err != nil {
t.Fatalf("Creating client error: %v", err)
}
var topic *TopicHandle
if topic, err = client.NewTopic(ctx, topicName); err != nil {
t.Errorf("CreateTopic error: %v", err)
}
var sub *SubscriptionHandle
if sub, err = topic.Subscribe(ctx, subName, nil); err != nil {
t.Errorf("CreateSub error: %v", err)
}
exists, err := topic.Exists(ctx)
if err != nil {
t.Fatalf("TopicExists error: %v", err)
}
if !exists {
t.Errorf("topic %s should exist, but it doesn't", topic)
}
exists, err = sub.Exists(ctx)
if err != nil {
t.Fatalf("SubExists error: %v", err)
}
if !exists {
t.Errorf("subscription %s should exist, but it doesn't", subName)
}
max := 10
msgs := make([]*Message, max)
expectedMsgs := make(map[string]bool, max)
for i := 0; i < max; i++ {
text := fmt.Sprintf("a message with an index %d", i)
attrs := make(map[string]string)
attrs["foo"] = "bar"
msgs[i] = &Message{
Data: []byte(text),
Attributes: attrs,
}
expectedMsgs[text] = false
}
ids, err := Publish(ctx, topicName, msgs...)
if err != nil {
t.Fatalf("Publish (1) error: %v", err)
}
if len(ids) != max {
t.Errorf("unexpected number of message IDs received; %d, want %d", len(ids), max)
}
expectedIDs := make(map[string]bool, max)
for _, id := range ids {
expectedIDs[id] = false
}
received, err := PullWait(ctx, subName, max)
if err != nil {
t.Fatalf("PullWait error: %v", err)
}
if len(received) != max {
t.Errorf("unexpected number of messages received; %d, want %d", len(received), max)
}
for _, msg := range received {
expectedMsgs[string(msg.Data)] = true
expectedIDs[msg.ID] = true
if msg.Attributes["foo"] != "bar" {
t.Errorf("message attribute foo is expected to be 'bar', found '%s'", msg.Attributes["foo"])
}
}
for msg, found := range expectedMsgs {
if !found {
t.Errorf("message '%s' should be received", msg)
}
}
for id, found := range expectedIDs {
if !found {
t.Errorf("message with the message id '%s' should be received", id)
}
}
// base64 test
data := "=@~"
msg := &Message{
Data: []byte(data),
}
_, err = Publish(ctx, topicName, msg)
if err != nil {
t.Fatalf("Publish (2) error: %v", err)
}
received, err = PullWait(ctx, subName, 1)
if err != nil {
t.Fatalf("PullWait error: %v", err)
}
if len(received) != 1 {
t.Fatalf("unexpected number of messages received; %d, want %d", len(received), 1)
}
if string(received[0].Data) != data {
t.Errorf("unexpexted message received; %s, want %s", string(received[0].Data), data)
}
err = sub.Delete(ctx)
if err != nil {
t.Errorf("DeleteSub error: %v", err)
}
err = topic.Delete(ctx)
if err != nil {
t.Errorf("DeleteTopic error: %v", err)
}
}
// uploadDockerImage makes a tar.gz snapshot of the camlistored docker image,
// and uploads it at camlistore-release/docker/camlistored-REV.tar.gz. It then
// makes a copy in the same bucket and path as camlistored.tar.gz.
func uploadDockerImage() {
proj := "camlistore-website"
bucket := "camlistore-release"
versionedTarball := "docker/camlistored-" + rev() + ".tar.gz"
tarball := "docker/camlistored.tar.gz"
log.Printf("Uploading %s/%s ...", bucket, versionedTarball)
ts, err := tokenSource(bucket)
if err != nil {
log.Fatal(err)
}
ctx := context.Background()
stoClient, err := storage.NewClient(ctx, cloud.WithTokenSource(ts), cloud.WithBaseHTTP(oauth2.NewClient(ctx, ts)))
if err != nil {
log.Fatal(err)
}
w := stoClient.Bucket(bucket).Object(versionedTarball).NewWriter(ctx)
w.ACL = publicACL(proj)
w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations?
w.ContentType = "application/x-gtar"
dockerSave := exec.Command("docker", "save", serverImage)
dockerSave.Stderr = os.Stderr
tar, err := dockerSave.StdoutPipe()
if err != nil {
log.Fatal(err)
}
targz, pw := io.Pipe()
go func() {
zw := gzip.NewWriter(pw)
n, err := io.Copy(zw, tar)
if err != nil {
log.Fatalf("Error copying to gzip writer: after %d bytes, %v", n, err)
}
if err := zw.Close(); err != nil {
log.Fatalf("gzip.Close: %v", err)
}
pw.CloseWithError(err)
}()
if err := dockerSave.Start(); err != nil {
log.Fatalf("Error starting docker save %v: %v", serverImage, err)
}
if _, err := io.Copy(w, targz); err != nil {
log.Fatalf("io.Copy: %v", err)
}
if err := w.Close(); err != nil {
log.Fatalf("closing GCS storage writer: %v", err)
}
if err := dockerSave.Wait(); err != nil {
log.Fatalf("Error waiting for docker save %v: %v", serverImage, err)
}
log.Printf("Uploaded tarball to %s", versionedTarball)
if !isWIP() {
log.Printf("Copying tarball to %s/%s ...", bucket, tarball)
if _, err := stoClient.CopyObject(ctx,
bucket, versionedTarball,
bucket, tarball,
&storage.ObjectAttrs{
ACL: publicACL(proj),
CacheControl: "no-cache",
ContentType: "application/x-gtar",
}); err != nil {
log.Fatalf("Error uploading %v: %v", tarball, err)
}
log.Printf("Uploaded tarball to %s", tarball)
}
}