func TestAdminClient(t *testing.T) {
ctx := context.Background()
projectID := testutil.ProjID()
newBucket := projectID + "copy"
client, err := NewAdminClient(ctx, projectID, cloud.WithTokenSource(testutil.TokenSource(ctx, ScopeFullControl)))
if err != nil {
t.Fatalf("Could not create client: %v", err)
}
defer client.Close()
if err := client.CreateBucket(ctx, newBucket, nil); err != nil {
t.Errorf("CreateBucket(%v, %v) failed %v", newBucket, nil, err)
}
if err := client.DeleteBucket(ctx, newBucket); err != nil {
t.Errorf("DeleteBucket(%v) failed %v", newBucket, err)
t.Logf("TODO: Warning this test left a new bucket in the cloud project, it must be deleted manually")
}
attrs := BucketAttrs{
DefaultObjectACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}},
}
if err := client.CreateBucket(ctx, newBucket, &attrs); err != nil {
t.Errorf("CreateBucket(%v, %v) failed %v", newBucket, attrs, err)
}
if err := client.DeleteBucket(ctx, newBucket); err != nil {
t.Errorf("DeleteBucket(%v) failed %v", newBucket, err)
t.Logf("TODO: Warning this test left a new bucket in the cloud project, it must be deleted manually")
}
}
func TestIntegrationPingBadProject(t *testing.T) {
if testing.Short() {
t.Skip("Integration tests skipped in short mode")
}
ctx := context.Background()
ts := testutil.TokenSource(ctx, Scope)
if ts == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
for _, projID := range []string{
testutil.ProjID() + "-BAD", // nonexistent project
"amazing-height-519", // exists, but wrong creds
} {
c, err := NewClient(ctx, projID, "logging-integration-test", cloud.WithTokenSource(ts))
if err != nil {
t.Fatalf("project %s: error creating client: %v", projID, err)
}
if err := c.Ping(); err == nil {
t.Errorf("project %s: want error pinging logging api, got nil", projID)
}
// Ping twice, just to make sure the deduping doesn't mess with the result.
if err := c.Ping(); err == nil {
t.Errorf("project %s: want error pinging logging api, got nil", projID)
}
}
}
func newClient(ctx context.Context) *Client {
ts := testutil.TokenSource(ctx, ScopeDatastore, ScopeUserEmail)
client, err := NewClient(ctx, testutil.ProjID(), cloud.WithTokenSource(ts))
if err != nil {
log.Fatal(err)
}
return client
}
func newClient(ctx context.Context, t *testing.T) *Client {
ts := testutil.TokenSource(ctx, ScopeDatastore)
if ts == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
client, err := NewClient(ctx, testutil.ProjID(), cloud.WithTokenSource(ts))
if err != nil {
t.Fatalf("NewClient: %v", err)
}
return client
}
// config is like testConfig, but it doesn't need a *testing.T.
func config(ctx context.Context) (*Client, string) {
ts := testutil.TokenSource(ctx, ScopeFullControl)
if ts == nil {
return nil, ""
}
p := testutil.ProjID()
if p == "" {
log.Fatal("The project ID must be set. See CONTRIBUTING.md for details")
}
client, err := NewClient(ctx, cloud.WithTokenSource(ts))
if err != nil {
log.Fatalf("NewClient: %v", err)
}
return client, p
}
// testConfig returns the Client used to access GCS and the default bucket
// name to use.
func testConfig(ctx context.Context, t *testing.T) (*Client, string) {
ts := cloud.WithTokenSource(testutil.TokenSource(ctx, ScopeFullControl))
if ts == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
p := testutil.ProjID()
if p == "" {
log.Fatal("The project ID must be set. See CONTRIBUTING.md for details")
}
client, err := NewClient(ctx, ts)
if err != nil {
t.Fatalf("NewClient: %v", err)
}
return client, p
}
func TestIntegration(t *testing.T) {
if testing.Short() {
t.Skip("Integration tests skipped in short mode")
}
ctx := context.Background()
ts := testutil.TokenSource(ctx, Scope)
if ts == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
projID := testutil.ProjID()
c, err := NewClient(ctx, projID, "logging-integration-test", cloud.WithTokenSource(ts))
if err != nil {
t.Fatalf("error creating client: %v", err)
}
if err := c.Ping(); err != nil {
t.Fatalf("error pinging logging api: %v", err)
}
// Ping twice, to verify that deduping doesn't change the result.
if err := c.Ping(); err != nil {
t.Fatalf("error pinging logging api: %v", err)
}
if err := c.LogSync(Entry{Payload: customJSONObject{}}); err != nil {
t.Fatalf("error writing log: %v", err)
}
if err := c.Log(Entry{Payload: customJSONObject{}}); err != nil {
t.Fatalf("error writing log: %v", err)
}
if _, err := c.Writer(Default).Write([]byte("test log with io.Writer")); err != nil {
t.Fatalf("error writing log using io.Writer: %v", err)
}
c.Logger(Default).Println("test log with log.Logger")
if err := c.Flush(); err != nil {
t.Fatalf("error flushing logs: %v", err)
}
}
func TestAdminClient(t *testing.T) {
if testing.Short() {
t.Skip("Integration tests skipped in short mode")
}
ctx := context.Background()
ts := testutil.TokenSource(ctx, ScopeFullControl)
if ts == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
projectID := testutil.ProjID()
newBucket := projectID + suffix
t.Logf("Testing admin with Bucket %q", newBucket)
client, err := NewAdminClient(ctx, projectID, cloud.WithTokenSource(ts))
if err != nil {
t.Fatalf("Could not create client: %v", err)
}
defer client.Close()
if err := client.CreateBucket(ctx, newBucket, nil); err != nil {
t.Errorf("CreateBucket(%v, %v) failed %v", newBucket, nil, err)
}
if err := client.DeleteBucket(ctx, newBucket); err != nil {
t.Errorf("DeleteBucket(%v) failed %v", newBucket, err)
t.Logf("TODO: Warning this test left a new bucket in the cloud project, it must be deleted manually")
}
attrs := BucketAttrs{
DefaultObjectACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}},
}
if err := client.CreateBucket(ctx, newBucket, &attrs); err != nil {
t.Errorf("CreateBucket(%v, %v) failed %v", newBucket, attrs, err)
}
if err := client.DeleteBucket(ctx, newBucket); err != nil {
t.Errorf("DeleteBucket(%v) failed %v", newBucket, err)
t.Logf("TODO: Warning this test left a new bucket in the cloud project, it must be deleted manually")
}
}
func TestAll(t *testing.T) {
if testing.Short() {
t.Skip("Integration tests skipped in short mode")
}
// TODO(djd): Replace this ctx with context.Background() when the new API is complete.
ctx := testutil.Context(ScopePubSub, ScopeCloudPlatform)
if ctx == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
ts := testutil.TokenSource(ctx, ScopePubSub, ScopeCloudPlatform)
if ts == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
now := time.Now()
topicName := fmt.Sprintf("topic-%d", now.Unix())
subName := fmt.Sprintf("subscription-%d", now.Unix())
client, err := NewClient(ctx, testutil.ProjID(), cloud.WithTokenSource(ts))
if err != nil {
t.Fatalf("Creating client error: %v", err)
}
var topic *TopicHandle
if topic, err = client.NewTopic(ctx, topicName); err != nil {
t.Errorf("CreateTopic error: %v", err)
}
var sub *SubscriptionHandle
if sub, err = topic.Subscribe(ctx, subName, nil); err != nil {
t.Errorf("CreateSub error: %v", err)
}
exists, err := topic.Exists(ctx)
if err != nil {
t.Fatalf("TopicExists error: %v", err)
}
if !exists {
t.Errorf("topic %s should exist, but it doesn't", topic)
}
exists, err = sub.Exists(ctx)
if err != nil {
t.Fatalf("SubExists error: %v", err)
}
if !exists {
t.Errorf("subscription %s should exist, but it doesn't", subName)
}
max := 10
msgs := make([]*Message, max)
expectedMsgs := make(map[string]bool, max)
for i := 0; i < max; i++ {
text := fmt.Sprintf("a message with an index %d", i)
attrs := make(map[string]string)
attrs["foo"] = "bar"
msgs[i] = &Message{
Data: []byte(text),
Attributes: attrs,
}
expectedMsgs[text] = false
}
ids, err := Publish(ctx, topicName, msgs...)
if err != nil {
t.Fatalf("Publish (1) error: %v", err)
}
if len(ids) != max {
t.Errorf("unexpected number of message IDs received; %d, want %d", len(ids), max)
}
expectedIDs := make(map[string]bool, max)
for _, id := range ids {
expectedIDs[id] = false
}
received, err := PullWait(ctx, subName, max)
if err != nil {
t.Fatalf("PullWait error: %v", err)
}
if len(received) != max {
t.Errorf("unexpected number of messages received; %d, want %d", len(received), max)
}
for _, msg := range received {
expectedMsgs[string(msg.Data)] = true
expectedIDs[msg.ID] = true
if msg.Attributes["foo"] != "bar" {
t.Errorf("message attribute foo is expected to be 'bar', found '%s'", msg.Attributes["foo"])
}
}
for msg, found := range expectedMsgs {
if !found {
t.Errorf("message '%s' should be received", msg)
}
}
for id, found := range expectedIDs {
if !found {
t.Errorf("message with the message id '%s' should be received", id)
}
}
// base64 test
data := "=@~"
msg := &Message{
Data: []byte(data),
}
_, err = Publish(ctx, topicName, msg)
if err != nil {
t.Fatalf("Publish (2) error: %v", err)
}
received, err = PullWait(ctx, subName, 1)
if err != nil {
t.Fatalf("PullWait error: %v", err)
}
if len(received) != 1 {
t.Fatalf("unexpected number of messages received; %d, want %d", len(received), 1)
}
if string(received[0].Data) != data {
t.Errorf("unexpexted message received; %s, want %s", string(received[0].Data), data)
}
err = sub.Delete(ctx)
if err != nil {
t.Errorf("DeleteSub error: %v", err)
}
err = topic.Delete(ctx)
if err != nil {
t.Errorf("DeleteTopic error: %v", err)
}
}
func TestAll(t *testing.T) {
if testing.Short() {
t.Skip("Integration tests skipped in short mode")
}
ctx := context.Background()
ts := testutil.TokenSource(ctx, ScopePubSub, ScopeCloudPlatform)
if ts == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
now := time.Now()
topicName := fmt.Sprintf("topic-%d", now.Unix())
subName := fmt.Sprintf("subscription-%d", now.Unix())
client, err := NewClient(ctx, testutil.ProjID(), cloud.WithTokenSource(ts))
if err != nil {
t.Fatalf("Creating client error: %v", err)
}
var topic *Topic
if topic, err = client.NewTopic(ctx, topicName); err != nil {
t.Errorf("CreateTopic error: %v", err)
}
var sub *Subscription
if sub, err = client.NewSubscription(ctx, subName, topic, 0, nil); err != nil {
t.Errorf("CreateSub error: %v", err)
}
exists, err := topic.Exists(ctx)
if err != nil {
t.Fatalf("TopicExists error: %v", err)
}
if !exists {
t.Errorf("topic %s should exist, but it doesn't", topic)
}
exists, err = sub.Exists(ctx)
if err != nil {
t.Fatalf("SubExists error: %v", err)
}
if !exists {
t.Errorf("subscription %s should exist, but it doesn't", subName)
}
msgs := []*Message{}
for i := 0; i < 10; i++ {
text := fmt.Sprintf("a message with an index %d", i)
attrs := make(map[string]string)
attrs["foo"] = "bar"
msgs = append(msgs, &Message{
Data: []byte(text),
Attributes: attrs,
})
}
ids, err := topic.Publish(ctx, msgs...)
if err != nil {
t.Fatalf("Publish (1) error: %v", err)
}
if len(ids) != len(msgs) {
t.Errorf("unexpected number of message IDs received; %d, want %d", len(ids), len(msgs))
}
want := make(map[string]*messageData)
for i, m := range msgs {
md := extractMessageData(m)
md.ID = ids[i]
want[md.ID] = md
}
// Use a timeout to ensure that Pull does not block indefinitely if there are unexpectedly few messages available.
timeoutCtx, _ := context.WithTimeout(ctx, time.Minute)
it, err := sub.Pull(timeoutCtx)
if err != nil {
t.Fatalf("error constructing iterator: %v", err)
}
defer it.Stop()
got := make(map[string]*messageData)
for i := 0; i < len(want); i++ {
m, err := it.Next()
if err != nil {
t.Fatalf("error getting next message: %v", err)
}
md := extractMessageData(m)
got[md.ID] = md
m.Done(true)
}
if !reflect.DeepEqual(got, want) {
t.Errorf("messages: got: %v ; want: %v", got, want)
}
// base64 test
data := "=@~"
_, err = topic.Publish(ctx, &Message{Data: []byte(data)})
if err != nil {
t.Fatalf("Publish error: %v", err)
}
m, err := it.Next()
if err != nil {
t.Fatalf("Pull error: %v", err)
}
if string(m.Data) != data {
t.Errorf("unexpected message received; %s, want %s", string(m.Data), data)
}
m.Done(true)
err = sub.Delete(ctx)
if err != nil {
t.Errorf("DeleteSub error: %v", err)
}
err = topic.Delete(ctx)
if err != nil {
t.Errorf("DeleteTopic error: %v", err)
}
}
// TestEndToEnd pumps many messages into a topic and tests that they are all delivered to each subscription for the topic.
// It also tests that messages are not unexpectedly redelivered.
func TestEndToEnd(t *testing.T) {
if testing.Short() {
t.Skip("Integration tests skipped in short mode")
}
ctx := context.Background()
ts := testutil.TokenSource(ctx, ScopePubSub, ScopeCloudPlatform)
if ts == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
now := time.Now()
topicName := fmt.Sprintf("endtoend-%d", now.Unix())
subPrefix := fmt.Sprintf("endtoend-%d", now.Unix())
client, err := NewClient(ctx, testutil.ProjID(), cloud.WithTokenSource(ts))
if err != nil {
t.Fatalf("Creating client error: %v", err)
}
var topic *Topic
if topic, err = client.NewTopic(ctx, topicName); err != nil {
t.Fatalf("CreateTopic error: %v", err)
}
defer topic.Delete(ctx)
// Three subscriptions to the same topic.
var subA, subB, subC *Subscription
if subA, err = client.NewSubscription(ctx, subPrefix+"-a", topic, ackDeadline, nil); err != nil {
t.Fatalf("CreateSub error: %v", err)
}
defer subA.Delete(ctx)
if subB, err = client.NewSubscription(ctx, subPrefix+"-b", topic, ackDeadline, nil); err != nil {
t.Fatalf("CreateSub error: %v", err)
}
defer subB.Delete(ctx)
if subC, err = client.NewSubscription(ctx, subPrefix+"-c", topic, ackDeadline, nil); err != nil {
t.Fatalf("CreateSub error: %v", err)
}
defer subC.Delete(ctx)
expectedCounts := make(map[string]int)
for _, id := range publish(t, ctx, topic) {
expectedCounts[id] = 1
}
// recv provides an indication that messages are still arriving.
recv := make(chan struct{})
// Keep track of the number of times each message (by message id) was
// seen from each subscription.
mcA := &messageCounter{counts: make(map[string]int), recv: recv}
mcB := &messageCounter{counts: make(map[string]int), recv: recv}
mcC := &messageCounter{counts: make(map[string]int), recv: recv}
stopC := make(chan struct{})
// We have three subscriptions to our topic.
// Each subscription will get a copy of each pulished message.
//
// subA has just one iterator, while subB has two. The subB iterators
// will each process roughly half of the messages for subB. All of
// these iterators live until all messages have been consumed. subC is
// processed by a series of short-lived iterators.
var wg sync.WaitGroup
con := &consumer{
concurrencyPerIterator: 1,
iteratorsInFlight: 2,
lifetimes: immortal,
}
con.consume(t, ctx, subA, mcA, &wg, stopC)
con = &consumer{
concurrencyPerIterator: 1,
iteratorsInFlight: 2,
lifetimes: immortal,
}
con.consume(t, ctx, subB, mcB, &wg, stopC)
con = &consumer{
concurrencyPerIterator: 1,
iteratorsInFlight: 2,
lifetimes: &explicitLifetimes{
lifetimes: []time.Duration{ackDeadline, ackDeadline, ackDeadline / 2, ackDeadline / 2},
},
}
con.consume(t, ctx, subC, mcC, &wg, stopC)
go func() {
timeoutC := time.After(timeout)
// Every time this ticker ticks, we will check if we have received any
// messages since the last time it ticked. We check less frequently
// than the ack deadline, so that we can detect if messages are
// redelivered after having their ack deadline extended.
checkQuiescence := time.NewTicker(ackDeadline * 3)
defer checkQuiescence.Stop()
var received bool
for {
select {
case <-recv:
received = true
case <-checkQuiescence.C:
if received {
received = false
} else {
close(stopC)
return
}
case <-timeoutC:
t.Errorf("timed out")
close(stopC)
return
}
}
}()
wg.Wait()
for _, mc := range []*messageCounter{mcA, mcB, mcC} {
if got, want := mc.counts, expectedCounts; !reflect.DeepEqual(got, want) {
t.Errorf("message counts: %v\n", diff(got, want))
}
}
}