func TestAdminClient(t *testing.T) { ctx := context.Background() projectID := testutil.ProjID() newBucket := projectID + "copy" client, err := NewAdminClient(ctx, projectID, cloud.WithTokenSource(testutil.TokenSource(ctx, ScopeFullControl))) if err != nil { t.Fatalf("Could not create client: %v", err) } defer client.Close() if err := client.CreateBucket(ctx, newBucket, nil); err != nil { t.Errorf("CreateBucket(%v, %v) failed %v", newBucket, nil, err) } if err := client.DeleteBucket(ctx, newBucket); err != nil { t.Errorf("DeleteBucket(%v) failed %v", newBucket, err) t.Logf("TODO: Warning this test left a new bucket in the cloud project, it must be deleted manually") } attrs := BucketAttrs{ DefaultObjectACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}}, } if err := client.CreateBucket(ctx, newBucket, &attrs); err != nil { t.Errorf("CreateBucket(%v, %v) failed %v", newBucket, attrs, err) } if err := client.DeleteBucket(ctx, newBucket); err != nil { t.Errorf("DeleteBucket(%v) failed %v", newBucket, err) t.Logf("TODO: Warning this test left a new bucket in the cloud project, it must be deleted manually") } }
func TestIntegrationPingBadProject(t *testing.T) { if testing.Short() { t.Skip("Integration tests skipped in short mode") } ctx := context.Background() ts := testutil.TokenSource(ctx, Scope) if ts == nil { t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") } for _, projID := range []string{ testutil.ProjID() + "-BAD", // nonexistent project "amazing-height-519", // exists, but wrong creds } { c, err := NewClient(ctx, projID, "logging-integration-test", cloud.WithTokenSource(ts)) if err != nil { t.Fatalf("project %s: error creating client: %v", projID, err) } if err := c.Ping(); err == nil { t.Errorf("project %s: want error pinging logging api, got nil", projID) } // Ping twice, just to make sure the deduping doesn't mess with the result. if err := c.Ping(); err == nil { t.Errorf("project %s: want error pinging logging api, got nil", projID) } } }
func newClient(ctx context.Context) *Client { ts := testutil.TokenSource(ctx, ScopeDatastore, ScopeUserEmail) client, err := NewClient(ctx, testutil.ProjID(), cloud.WithTokenSource(ts)) if err != nil { log.Fatal(err) } return client }
func newClient(ctx context.Context, t *testing.T) *Client { ts := testutil.TokenSource(ctx, ScopeDatastore) if ts == nil { t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") } client, err := NewClient(ctx, testutil.ProjID(), cloud.WithTokenSource(ts)) if err != nil { t.Fatalf("NewClient: %v", err) } return client }
// config is like testConfig, but it doesn't need a *testing.T. func config(ctx context.Context) (*Client, string) { ts := testutil.TokenSource(ctx, ScopeFullControl) if ts == nil { return nil, "" } p := testutil.ProjID() if p == "" { log.Fatal("The project ID must be set. See CONTRIBUTING.md for details") } client, err := NewClient(ctx, cloud.WithTokenSource(ts)) if err != nil { log.Fatalf("NewClient: %v", err) } return client, p }
// testConfig returns the Client used to access GCS and the default bucket // name to use. func testConfig(ctx context.Context, t *testing.T) (*Client, string) { ts := cloud.WithTokenSource(testutil.TokenSource(ctx, ScopeFullControl)) if ts == nil { t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") } p := testutil.ProjID() if p == "" { log.Fatal("The project ID must be set. See CONTRIBUTING.md for details") } client, err := NewClient(ctx, ts) if err != nil { t.Fatalf("NewClient: %v", err) } return client, p }
func TestIntegration(t *testing.T) { if testing.Short() { t.Skip("Integration tests skipped in short mode") } ctx := context.Background() ts := testutil.TokenSource(ctx, Scope) if ts == nil { t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") } projID := testutil.ProjID() c, err := NewClient(ctx, projID, "logging-integration-test", cloud.WithTokenSource(ts)) if err != nil { t.Fatalf("error creating client: %v", err) } if err := c.Ping(); err != nil { t.Fatalf("error pinging logging api: %v", err) } // Ping twice, to verify that deduping doesn't change the result. if err := c.Ping(); err != nil { t.Fatalf("error pinging logging api: %v", err) } if err := c.LogSync(Entry{Payload: customJSONObject{}}); err != nil { t.Fatalf("error writing log: %v", err) } if err := c.Log(Entry{Payload: customJSONObject{}}); err != nil { t.Fatalf("error writing log: %v", err) } if _, err := c.Writer(Default).Write([]byte("test log with io.Writer")); err != nil { t.Fatalf("error writing log using io.Writer: %v", err) } c.Logger(Default).Println("test log with log.Logger") if err := c.Flush(); err != nil { t.Fatalf("error flushing logs: %v", err) } }
func TestAdminClient(t *testing.T) { if testing.Short() { t.Skip("Integration tests skipped in short mode") } ctx := context.Background() ts := testutil.TokenSource(ctx, ScopeFullControl) if ts == nil { t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") } projectID := testutil.ProjID() newBucket := projectID + suffix t.Logf("Testing admin with Bucket %q", newBucket) client, err := NewAdminClient(ctx, projectID, cloud.WithTokenSource(ts)) if err != nil { t.Fatalf("Could not create client: %v", err) } defer client.Close() if err := client.CreateBucket(ctx, newBucket, nil); err != nil { t.Errorf("CreateBucket(%v, %v) failed %v", newBucket, nil, err) } if err := client.DeleteBucket(ctx, newBucket); err != nil { t.Errorf("DeleteBucket(%v) failed %v", newBucket, err) t.Logf("TODO: Warning this test left a new bucket in the cloud project, it must be deleted manually") } attrs := BucketAttrs{ DefaultObjectACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}}, } if err := client.CreateBucket(ctx, newBucket, &attrs); err != nil { t.Errorf("CreateBucket(%v, %v) failed %v", newBucket, attrs, err) } if err := client.DeleteBucket(ctx, newBucket); err != nil { t.Errorf("DeleteBucket(%v) failed %v", newBucket, err) t.Logf("TODO: Warning this test left a new bucket in the cloud project, it must be deleted manually") } }
func TestAll(t *testing.T) { if testing.Short() { t.Skip("Integration tests skipped in short mode") } // TODO(djd): Replace this ctx with context.Background() when the new API is complete. ctx := testutil.Context(ScopePubSub, ScopeCloudPlatform) if ctx == nil { t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") } ts := testutil.TokenSource(ctx, ScopePubSub, ScopeCloudPlatform) if ts == nil { t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") } now := time.Now() topicName := fmt.Sprintf("topic-%d", now.Unix()) subName := fmt.Sprintf("subscription-%d", now.Unix()) client, err := NewClient(ctx, testutil.ProjID(), cloud.WithTokenSource(ts)) if err != nil { t.Fatalf("Creating client error: %v", err) } var topic *TopicHandle if topic, err = client.NewTopic(ctx, topicName); err != nil { t.Errorf("CreateTopic error: %v", err) } var sub *SubscriptionHandle if sub, err = topic.Subscribe(ctx, subName, nil); err != nil { t.Errorf("CreateSub error: %v", err) } exists, err := topic.Exists(ctx) if err != nil { t.Fatalf("TopicExists error: %v", err) } if !exists { t.Errorf("topic %s should exist, but it doesn't", topic) } exists, err = sub.Exists(ctx) if err != nil { t.Fatalf("SubExists error: %v", err) } if !exists { t.Errorf("subscription %s should exist, but it doesn't", subName) } max := 10 msgs := make([]*Message, max) expectedMsgs := make(map[string]bool, max) for i := 0; i < max; i++ { text := fmt.Sprintf("a message with an index %d", i) attrs := make(map[string]string) attrs["foo"] = "bar" msgs[i] = &Message{ Data: []byte(text), Attributes: attrs, } expectedMsgs[text] = false } ids, err := Publish(ctx, topicName, msgs...) if err != nil { t.Fatalf("Publish (1) error: %v", err) } if len(ids) != max { t.Errorf("unexpected number of message IDs received; %d, want %d", len(ids), max) } expectedIDs := make(map[string]bool, max) for _, id := range ids { expectedIDs[id] = false } received, err := PullWait(ctx, subName, max) if err != nil { t.Fatalf("PullWait error: %v", err) } if len(received) != max { t.Errorf("unexpected number of messages received; %d, want %d", len(received), max) } for _, msg := range received { expectedMsgs[string(msg.Data)] = true expectedIDs[msg.ID] = true if msg.Attributes["foo"] != "bar" { t.Errorf("message attribute foo is expected to be 'bar', found '%s'", msg.Attributes["foo"]) } } for msg, found := range expectedMsgs { if !found { t.Errorf("message '%s' should be received", msg) } } for id, found := range expectedIDs { if !found { t.Errorf("message with the message id '%s' should be received", id) } } // base64 test data := "=@~" msg := &Message{ Data: []byte(data), } _, err = Publish(ctx, topicName, msg) if err != nil { t.Fatalf("Publish (2) error: %v", err) } received, err = PullWait(ctx, subName, 1) if err != nil { t.Fatalf("PullWait error: %v", err) } if len(received) != 1 { t.Fatalf("unexpected number of messages received; %d, want %d", len(received), 1) } if string(received[0].Data) != data { t.Errorf("unexpexted message received; %s, want %s", string(received[0].Data), data) } err = sub.Delete(ctx) if err != nil { t.Errorf("DeleteSub error: %v", err) } err = topic.Delete(ctx) if err != nil { t.Errorf("DeleteTopic error: %v", err) } }
func TestAll(t *testing.T) { if testing.Short() { t.Skip("Integration tests skipped in short mode") } ctx := context.Background() ts := testutil.TokenSource(ctx, ScopePubSub, ScopeCloudPlatform) if ts == nil { t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") } now := time.Now() topicName := fmt.Sprintf("topic-%d", now.Unix()) subName := fmt.Sprintf("subscription-%d", now.Unix()) client, err := NewClient(ctx, testutil.ProjID(), cloud.WithTokenSource(ts)) if err != nil { t.Fatalf("Creating client error: %v", err) } var topic *Topic if topic, err = client.NewTopic(ctx, topicName); err != nil { t.Errorf("CreateTopic error: %v", err) } var sub *Subscription if sub, err = client.NewSubscription(ctx, subName, topic, 0, nil); err != nil { t.Errorf("CreateSub error: %v", err) } exists, err := topic.Exists(ctx) if err != nil { t.Fatalf("TopicExists error: %v", err) } if !exists { t.Errorf("topic %s should exist, but it doesn't", topic) } exists, err = sub.Exists(ctx) if err != nil { t.Fatalf("SubExists error: %v", err) } if !exists { t.Errorf("subscription %s should exist, but it doesn't", subName) } msgs := []*Message{} for i := 0; i < 10; i++ { text := fmt.Sprintf("a message with an index %d", i) attrs := make(map[string]string) attrs["foo"] = "bar" msgs = append(msgs, &Message{ Data: []byte(text), Attributes: attrs, }) } ids, err := topic.Publish(ctx, msgs...) if err != nil { t.Fatalf("Publish (1) error: %v", err) } if len(ids) != len(msgs) { t.Errorf("unexpected number of message IDs received; %d, want %d", len(ids), len(msgs)) } want := make(map[string]*messageData) for i, m := range msgs { md := extractMessageData(m) md.ID = ids[i] want[md.ID] = md } // Use a timeout to ensure that Pull does not block indefinitely if there are unexpectedly few messages available. timeoutCtx, _ := context.WithTimeout(ctx, time.Minute) it, err := sub.Pull(timeoutCtx) if err != nil { t.Fatalf("error constructing iterator: %v", err) } defer it.Stop() got := make(map[string]*messageData) for i := 0; i < len(want); i++ { m, err := it.Next() if err != nil { t.Fatalf("error getting next message: %v", err) } md := extractMessageData(m) got[md.ID] = md m.Done(true) } if !reflect.DeepEqual(got, want) { t.Errorf("messages: got: %v ; want: %v", got, want) } // base64 test data := "=@~" _, err = topic.Publish(ctx, &Message{Data: []byte(data)}) if err != nil { t.Fatalf("Publish error: %v", err) } m, err := it.Next() if err != nil { t.Fatalf("Pull error: %v", err) } if string(m.Data) != data { t.Errorf("unexpected message received; %s, want %s", string(m.Data), data) } m.Done(true) err = sub.Delete(ctx) if err != nil { t.Errorf("DeleteSub error: %v", err) } err = topic.Delete(ctx) if err != nil { t.Errorf("DeleteTopic error: %v", err) } }
// TestEndToEnd pumps many messages into a topic and tests that they are all delivered to each subscription for the topic. // It also tests that messages are not unexpectedly redelivered. func TestEndToEnd(t *testing.T) { if testing.Short() { t.Skip("Integration tests skipped in short mode") } ctx := context.Background() ts := testutil.TokenSource(ctx, ScopePubSub, ScopeCloudPlatform) if ts == nil { t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") } now := time.Now() topicName := fmt.Sprintf("endtoend-%d", now.Unix()) subPrefix := fmt.Sprintf("endtoend-%d", now.Unix()) client, err := NewClient(ctx, testutil.ProjID(), cloud.WithTokenSource(ts)) if err != nil { t.Fatalf("Creating client error: %v", err) } var topic *Topic if topic, err = client.NewTopic(ctx, topicName); err != nil { t.Fatalf("CreateTopic error: %v", err) } defer topic.Delete(ctx) // Three subscriptions to the same topic. var subA, subB, subC *Subscription if subA, err = client.NewSubscription(ctx, subPrefix+"-a", topic, ackDeadline, nil); err != nil { t.Fatalf("CreateSub error: %v", err) } defer subA.Delete(ctx) if subB, err = client.NewSubscription(ctx, subPrefix+"-b", topic, ackDeadline, nil); err != nil { t.Fatalf("CreateSub error: %v", err) } defer subB.Delete(ctx) if subC, err = client.NewSubscription(ctx, subPrefix+"-c", topic, ackDeadline, nil); err != nil { t.Fatalf("CreateSub error: %v", err) } defer subC.Delete(ctx) expectedCounts := make(map[string]int) for _, id := range publish(t, ctx, topic) { expectedCounts[id] = 1 } // recv provides an indication that messages are still arriving. recv := make(chan struct{}) // Keep track of the number of times each message (by message id) was // seen from each subscription. mcA := &messageCounter{counts: make(map[string]int), recv: recv} mcB := &messageCounter{counts: make(map[string]int), recv: recv} mcC := &messageCounter{counts: make(map[string]int), recv: recv} stopC := make(chan struct{}) // We have three subscriptions to our topic. // Each subscription will get a copy of each pulished message. // // subA has just one iterator, while subB has two. The subB iterators // will each process roughly half of the messages for subB. All of // these iterators live until all messages have been consumed. subC is // processed by a series of short-lived iterators. var wg sync.WaitGroup con := &consumer{ concurrencyPerIterator: 1, iteratorsInFlight: 2, lifetimes: immortal, } con.consume(t, ctx, subA, mcA, &wg, stopC) con = &consumer{ concurrencyPerIterator: 1, iteratorsInFlight: 2, lifetimes: immortal, } con.consume(t, ctx, subB, mcB, &wg, stopC) con = &consumer{ concurrencyPerIterator: 1, iteratorsInFlight: 2, lifetimes: &explicitLifetimes{ lifetimes: []time.Duration{ackDeadline, ackDeadline, ackDeadline / 2, ackDeadline / 2}, }, } con.consume(t, ctx, subC, mcC, &wg, stopC) go func() { timeoutC := time.After(timeout) // Every time this ticker ticks, we will check if we have received any // messages since the last time it ticked. We check less frequently // than the ack deadline, so that we can detect if messages are // redelivered after having their ack deadline extended. checkQuiescence := time.NewTicker(ackDeadline * 3) defer checkQuiescence.Stop() var received bool for { select { case <-recv: received = true case <-checkQuiescence.C: if received { received = false } else { close(stopC) return } case <-timeoutC: t.Errorf("timed out") close(stopC) return } } }() wg.Wait() for _, mc := range []*messageCounter{mcA, mcB, mcC} { if got, want := mc.counts, expectedCounts; !reflect.DeepEqual(got, want) { t.Errorf("message counts: %v\n", diff(got, want)) } } }