// run executes the application.
func (a *application) run(ctx context.Context) error {
// Setup common context parameters.
ctx, cancelFunc := context.WithCancel(ctx)
ctx = retry.WithTransientOnly(retry.Use(ctx, exponentialBackoff))
// Monitor our shutdown singal. Cancel our context if it is closed.
go func() {
<-a.shutdownC
log.Infof(ctx, "Shutdown signal received; cancelling context.")
cancelFunc()
}()
wg := sync.WaitGroup{}
for i := 0; i < a.config.numWorkers; i++ {
i := i
wg.Add(1)
go func(id int) {
defer wg.Done()
a.process(log.SetField(ctx, "worker", i))
}(i)
}
wg.Wait()
return nil
}
func TestMain(t *testing.T) {
t.Parallel()
// Do not retry.
ctx := retry.Use(context.Background(), func(context.Context) retry.Iterator {
return &retry.Limited{}
})
Convey(`An application using testing stubs`, t, func() {
config := config{
pubsub: pubsubConfig{
project: "test-project",
topic: "test-topic",
subscription: "test-subscription",
create: false,
batchSize: 64,
},
endpoint: endpointConfig{
url: "fake-protocol://test.endpoint",
},
numWorkers: 10,
}
pubsubMock := &testPubSubService{
infinitePull: true,
}
endpointMock := &endpointServiceMock{
msgC: make(chan []byte),
}
pubsubMock.MockCall("SubExists", "test-subscription").WithResult(true, nil)
pubsubClient, err := newPubSubClient(ctx, config.pubsub, pubsubMock)
So(err, ShouldBeNil)
app := newApplication(config)
app.pubsub = pubsubClient
app.endpoint = endpointMock
// Run the application. Shut down after each test is complete.
finishedC := make(chan struct{})
go func() {
app.run(ctx)
close(finishedC)
}()
defer func() {
app.shutdown()
<-finishedC
So(pubsubMock, mock.ShouldHaveNoErrors)
So(endpointMock, mock.ShouldHaveNoErrors)
}()
Convey(`Will consume messages from Pub/Sub.`, func() {
buf := make([]byte, binary.MaxVarintLen64)
missing := make(map[int]bool)
for i := 0; i < 1024; i++ {
missing[i] = true
count := binary.PutUvarint(buf, uint64(i))
msg := &pubsub.Message{
ID: fmt.Sprintf("msg-%d", i),
AckID: fmt.Sprintf("ack-%d", i),
Data: make([]byte, count),
}
copy(msg.Data, buf)
pubsubMock.MockCall("Pull", "test-subscription", 64).WithResult([]*pubsub.Message{msg}, nil)
pubsubMock.MockCall("Ack", "test-subscription", []string{msg.AckID}).WithResult(nil)
endpointMock.MockCall("send", mock.Ignore, msg.Data).WithResult(nil)
}
missingCount := len(missing)
for i := 0; i < 1024; i++ {
msg := <-endpointMock.msgC
index, _ := binary.Uvarint(msg)
v := missing[int(index)]
So(v, ShouldBeTrue)
missing[int(index)] = false
missingCount--
}
So(missingCount, ShouldEqual, 0)
})
Convey(`Will refuse to process a message that is too large, and will ACK it.`, func() {
msgs := []*pubsub.Message{
{
ID: "msg-big",
AckID: "ack-big",
Data: bytes.Repeat([]byte{0xAA}, maxMessageSize+1),
},
{
ID: "msg-legit",
AckID: "ack-legit",
Data: bytes.Repeat([]byte{0x55}, maxMessageSize),
},
}
pubsubMock.MockCall("Pull", "test-subscription", 64).WithResult(msgs, nil)
pubsubMock.MockCall("Ack", "test-subscription", []string{"ack-big", "ack-legit"}).WithResult(nil)
endpointMock.MockCall("send", mock.Ignore, msgs[1].Data).WithResult(nil)
So(<-endpointMock.msgC, ShouldResemble, msgs[1].Data)
})
})
}
func TestCloudLogging(t *testing.T) {
Convey(`A cloud logging instance using a test HTTP client/server`, t, func() {
ctx, _ := testclock.UseTime(context.Background(), time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC))
// Do not retry.
ctx = retry.Use(ctx, func(context.Context) retry.Iterator {
return &retry.Limited{}
})
h := &testCloudLoggingHandler{
logC: make(chan *cloudLogBundle, 1),
}
srv := httptest.NewServer(h)
defer srv.Close()
tr := &http.Transport{
DialTLS: func(network, addr string) (net.Conn, error) {
u, err := url.Parse(srv.URL)
if err != nil {
return nil, err
}
return net.Dial(network, u.Host)
},
}
client := &http.Client{
Transport: tr,
}
config := loggerConfig{
projectName: "test-project",
resourceType: "test-resource",
logsID: "test-logs-id",
}
service, err := cloudlog.New(client)
So(err, ShouldBeNil)
cl := newCloudLogging(ctx, &config, service)
defer cl.finish()
Convey(`A bound cloud logging instance`, func() {
l := cl.bind(ctx)
Convey(`Can publish logging data.`, func() {
l.Infof("Message at %s", "INFO")
bundle := <-h.logC
So(len(bundle.Entries), ShouldEqual, 1)
So(bundle.Entries[0], shouldMatchLog, &cloudlog.LogEntry{
InsertId: "-0-0",
Metadata: &cloudlog.LogEntryMetadata{
ProjectId: "test-project",
Severity: "INFO",
Timestamp: "2015-01-01T00:00:00Z",
},
TextPayload: "Message at INFO",
})
})
Convey(`Will batch logging data.`, func() {
cl.testLogAckC = make(chan []*logEntry, 1)
// The first message will be read immediately.
l.Infof("Initial unbatched message.")
<-cl.testLogAckC
// The next set of messages will be batched, since we're not release our
// HTTP server yet.
for i := 0; i < cloudLoggingBatchSize; i++ {
l.Infof("Batch message #%d", i)
}
<-cl.testLogAckC
// Read the first bundle.
bundle := <-h.logC
So(len(bundle.Entries), ShouldEqual, 1)
So(bundle.Entries[0], shouldMatchLog, &cloudlog.LogEntry{
InsertId: "-0-0",
Metadata: &cloudlog.LogEntryMetadata{
ProjectId: "test-project",
Severity: "INFO",
Timestamp: "2015-01-01T00:00:00Z",
},
TextPayload: "Initial unbatched message.",
})
// Read the second bundle.
bundle = <-h.logC
So(len(bundle.Entries), ShouldEqual, cloudLoggingBatchSize)
for i, entry := range bundle.Entries {
So(entry, shouldMatchLog, &cloudlog.LogEntry{
InsertId: fmt.Sprintf("-1-%d", i),
Metadata: &cloudlog.LogEntryMetadata{
ProjectId: "test-project",
Severity: "INFO",
Timestamp: "2015-01-01T00:00:00Z",
},
TextPayload: fmt.Sprintf("Batch message #%d", i),
})
}
})
})
})
}
func TestPubSub(t *testing.T) {
t.Parallel()
Convey(`Using a testing Pub/Sub config`, t, func() {
// Do not retry.
ctx := retry.Use(context.Background(), func(context.Context) retry.Iterator {
return &retry.Limited{}
})
config := pubsubConfig{
project: "test-project",
topic: "test-topic",
subscription: "test-subscription",
create: true,
batchSize: 64,
}
svc := &testPubSubService{}
defer So(svc, mock.ShouldHaveNoErrors)
Convey(`When the subscription does not exist`, func() {
svc.MockCall("SubExists", "test-subscription").WithResult(false, nil)
Convey(`And the topic does not exist, will create a new topic and subscription.`, func() {
svc.MockCall("TopicExists", "test-topic").WithResult(false, nil)
svc.MockCall("CreateTopic", "test-topic").WithResult(nil)
svc.MockCall("CreatePullSub", "test-subscription", "test-topic").WithResult(nil)
_, err := newPubSubClient(ctx, config, svc)
So(err, ShouldBeNil)
})
Convey(`And the topic exists, will create a new subscription.`, func() {
svc.MockCall("TopicExists", "test-topic").WithResult(true, nil)
svc.MockCall("CreatePullSub", "test-subscription", "test-topic").WithResult(nil)
_, err := newPubSubClient(ctx, config, svc)
So(err, ShouldBeNil)
})
Convey(`Will fail to create a new client when "create" is false.`, func() {
config.create = false
_, err := newPubSubClient(ctx, config, svc)
So(err, ShouldNotBeNil)
})
})
Convey(`Will create a new client.`, func() {
svc.MockCall("SubExists", "test-subscription").WithResult(true, nil)
client, err := newPubSubClient(ctx, config, svc)
So(err, ShouldBeNil)
Convey(`When executing pull/ack with no messages`, func() {
svc.MockCall("Pull", "test-subscription", 64).WithResult(nil, nil)
Convey(`Returns errNoMessages.`, func() {
err := client.pullAckMessages(ctx, func([]*pubsub.Message) {})
So(err, ShouldEqual, errNoMessages)
})
})
Convey(`When executing pull/ack with one message`, func() {
msgs := []*pubsub.Message{
{
ID: "id0",
AckID: "ack0",
Data: []byte{0xd0, 0x65},
},
}
svc.MockCall("Pull", "test-subscription", 64).WithResult(msgs, nil)
Convey(`Returns and ACKs that message.`, func() {
svc.MockCall("Ack", "test-subscription", []string{"ack0"}).WithResult(nil)
var pullMsg []*pubsub.Message
err := client.pullAckMessages(ctx, func(msg []*pubsub.Message) {
pullMsg = msg
})
So(err, ShouldBeNil)
So(pullMsg, ShouldResemble, msgs)
})
Convey(`ACKs the message even if the handler panics.`, func() {
svc.MockCall("Ack", "test-subscription", []string{"ack0"}).WithResult(nil)
So(func() {
client.pullAckMessages(ctx, func(msg []*pubsub.Message) {
panic("Handler failure!")
})
}, ShouldPanic)
})
Convey(`Does not ACK the message if the handler clears it.`, func() {
err := client.pullAckMessages(ctx, func(msg []*pubsub.Message) {
for i := range msg {
msg[i] = nil
}
})
So(err, ShouldBeNil)
})
})
Convey(`When executing pull/ack with an error`, func() {
e := errors.New("TEST ERROR")
svc.MockCall("Pull", "test-subscription", 64).WithResult(nil, e)
Convey(`Returns the error as transient.`, func() {
So(client.pullAckMessages(ctx, func([]*pubsub.Message) {}), ShouldResemble, luciErrors.Transient{Err: e})
})
})
})
})
}
// TestEndpoint tests the endpoint implementation and API.
func TestEndpointService(t *testing.T) {
t.Parallel()
Convey(`An endpoint service connected to a testing HTTP server`, t, func() {
ctx, tc := testclock.UseTime(context.Background(), time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC))
// Retry up to ten times without delaying.
ctx = retry.Use(ctx, func(context.Context) retry.Iterator {
return &retry.Limited{Retries: 10}
})
h := &testEndpointServiceHandler{}
srv := httptest.NewTLSServer(h)
defer srv.Close()
tr := &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
}
client := &http.Client{
Transport: tr,
}
c := &endpointServiceImpl{
endpointConfig: endpointConfig{url: srv.URL},
client: client,
}
msg := bytes.Repeat([]byte{0x60, 0x0d, 0xd0, 0x65}, 32)
Convey(`Successfully posts a message.`, func() {
So(c.send(ctx, msg), ShouldBeNil)
So(h.connections, ShouldEqual, 1)
So(h.errors, ShouldEqual, 0)
So(h.messages, ShouldResemble, [][]byte{msg})
})
Convey(`Retries sending when an error is encountered.`, func() {
tc.SetTimerCallback(func(t clock.Timer) {
tc.Add(time.Second)
})
h.failures = 4
So(c.send(ctx, msg), ShouldBeNil)
So(h.connections, ShouldEqual, 5)
So(h.errors, ShouldEqual, 4)
So(h.messages, ShouldResemble, [][]byte{msg})
})
Convey(`Returns a transient error when a send completely fails.`, func() {
h.failures = 11
err := c.send(ctx, msg)
So(err, ShouldNotBeNil)
So(err, ShouldHaveSameTypeAs, luciErrors.Transient{})
So(h.connections, ShouldEqual, 11)
So(h.errors, ShouldEqual, 11)
So(h.messages, ShouldResemble, [][]byte(nil))
})
})
}